From 960092f0637e5911777784743789a8cad23ce4c1 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Tue, 3 Dec 2024 14:25:44 +1000 Subject: [PATCH 1/2] upgrade prometheus and fix tests Signed-off-by: Gabriel Fukushima --- .../validator/coordinator/Eth1DataCache.java | 2 +- .../PrometheusMetricsPublisherSource.java | 22 +- .../generators/CachingTaskQueue.java | 6 +- .../generators/CachingTaskQueueTest.java | 20 +- ...onLayerBlockProductionManagerImplTest.java | 8 +- .../ExecutionLayerManagerImplTest.java | 8 +- .../interop/MergedGenesisTestBuilder.java | 4 +- .../forkchoice/ProposersDataManager.java | 8 +- ...gregatingSignatureVerificationService.java | 11 +- .../forkchoice/ProposerDataManagerTest.java | 4 +- ...BlockBlobSidecarsTrackersPoolImplTest.java | 43 ++-- gradle/versions.gradle | 6 +- .../async/MetricTrackingExecutorFactory.java | 6 +- .../ThrottlingTaskQueueWithPriority.java | 6 +- infrastructure/metrics/build.gradle | 3 +- .../metrics/MetricsEndpoint.java | 5 +- .../metrics/MetricsHistogram.java | 199 +----------------- .../metrics/SettableLabelledGauge.java | 10 +- .../MetricsCountersByIntervalsTest.java | 57 +++-- .../metrics/MetricsHistogramTest.java | 83 -------- .../metrics/StubLabelledGauge.java | 4 +- .../metrics/StubMetricsSystem.java | 177 +++++++++++++++- .../gossip/AttestationGossipManagerTest.java | 11 +- ...BlobSidecarsByRangeMessageHandlerTest.java | 14 +- .../BlobSidecarsByRootMessageHandlerTest.java | 12 +- .../networking/p2p/libp2p/PeerManager.java | 10 +- .../p2p/libp2p/PeerManagerTest.java | 10 +- storage/build.gradle | 2 +- .../server/pruner/BlobSidecarPruner.java | 6 +- .../storage/server/rocksdb/RocksDbStats.java | 64 ++---- .../storage/client/RecentChainDataTest.java | 3 +- .../java/tech/pegasys/teku/AbstractNode.java | 2 +- ...etricRecordingValidatorApiChannelTest.java | 10 +- .../validator/client/PendingDutiesTest.java | 13 +- .../FailoverValidatorApiHandlerTest.java | 11 +- 35 files changed, 381 insertions(+), 479 deletions(-) delete mode 100644 infrastructure/metrics/src/test/java/tech/pegasys/teku/infrastructure/metrics/MetricsHistogramTest.java diff --git a/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/Eth1DataCache.java b/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/Eth1DataCache.java index d688d0e057e..e187caf39c4 100644 --- a/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/Eth1DataCache.java +++ b/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/Eth1DataCache.java @@ -35,7 +35,7 @@ public class Eth1DataCache { static final String CACHE_SIZE_METRIC_NAME = "eth1_block_cache_size"; static final String VOTES_MAX_METRIC_NAME = "eth1_current_period_votes_max"; - static final String VOTES_TOTAL_METRIC_NAME = "eth1_current_period_votes_total"; + static final String VOTES_TOTAL_METRIC_NAME = "eth1_current_period_votes"; static final String VOTES_UNKNOWN_METRIC_NAME = "eth1_current_period_votes_unknown"; static final String VOTES_CURRENT_METRIC_NAME = "eth1_current_period_votes_current"; static final String VOTES_BEST_METRIC_NAME = "eth1_current_period_votes_best"; diff --git a/data/publisher/src/main/java/tech/pegasys/teku/data/publisher/PrometheusMetricsPublisherSource.java b/data/publisher/src/main/java/tech/pegasys/teku/data/publisher/PrometheusMetricsPublisherSource.java index 4e538d402e2..73c380f8649 100644 --- a/data/publisher/src/main/java/tech/pegasys/teku/data/publisher/PrometheusMetricsPublisherSource.java +++ b/data/publisher/src/main/java/tech/pegasys/teku/data/publisher/PrometheusMetricsPublisherSource.java @@ -99,7 +99,7 @@ public long getGossipBytesTotalReceived() { } private void storeObservationIfNeeded(final Observation observation) { - MetricCategory category = observation.getCategory(); + MetricCategory category = observation.category(); if (category.equals(PROCESS)) { readProcessCategoryItem(observation); } else if (category.equals(VALIDATOR)) { @@ -113,37 +113,37 @@ private void storeObservationIfNeeded(final Observation observation) { private void readBeaconCategoryItem(final Observation observation) { isBeaconNodePresent = true; - switch (observation.getMetricName()) { + switch (observation.metricName()) { case "head_slot": - headSlot = getLongValue(observation.getValue()); + headSlot = getLongValue(observation.value()); break; case "eth1_request_queue_size": isEth1Connected = true; break; case "peer_count": - peerCount = getIntValue(observation.getValue()); + peerCount = getIntValue(observation.value()); break; case "node_syncing_active": - isEth2Synced = getIntValue(observation.getValue()) == 0; + isEth2Synced = getIntValue(observation.value()) == 0; break; } } private void readProcessCategoryItem(final Observation observation) { - if ("cpu_seconds_total".equals(observation.getMetricName())) { - cpuSecondsTotal = getLongValue(observation.getValue()); + if ("cpu_seconds_total".equals(observation.metricName())) { + cpuSecondsTotal = getLongValue(observation.value()); } } private void readJvmCategoryItem(final Observation observation) { - if ("memory_pool_bytes_used".equals(observation.getMetricName())) { - addToMemoryPoolBytesUsed((Double) observation.getValue()); + if ("memory_pool_bytes_used".equals(observation.metricName())) { + addToMemoryPoolBytesUsed((Double) observation.value()); } } private void readValidatorCategoryItem(final Observation observation) { - if ("local_validator_counts".equals(observation.getMetricName())) { - addToLocalValidators(observation.getLabels(), (Double) observation.getValue()); + if ("local_validator_counts".equals(observation.metricName())) { + addToLocalValidators(observation.labels(), (Double) observation.value()); } } diff --git a/ethereum/dataproviders/src/main/java/tech/pegasys/teku/dataproviders/generators/CachingTaskQueue.java b/ethereum/dataproviders/src/main/java/tech/pegasys/teku/dataproviders/generators/CachingTaskQueue.java index 7cb4d14f717..fb2a151cdf5 100644 --- a/ethereum/dataproviders/src/main/java/tech/pegasys/teku/dataproviders/generators/CachingTaskQueue.java +++ b/ethereum/dataproviders/src/main/java/tech/pegasys/teku/dataproviders/generators/CachingTaskQueue.java @@ -26,8 +26,8 @@ import java.util.stream.Stream; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; -import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; import tech.pegasys.teku.infrastructure.async.AsyncRunner; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.collections.LimitedMap; @@ -88,8 +88,8 @@ public static CachingTaskQueue create( } public void startMetrics() { - final LabelledGauge taskQueueMetrics = - metricsSystem.createLabelledGauge( + final LabelledSuppliedMetric taskQueueMetrics = + metricsSystem.createLabelledSuppliedGauge( TekuMetricCategory.STORAGE, metricsPrefix + "_tasks", "Labelled task queue metrics", diff --git a/ethereum/dataproviders/src/test/java/tech/pegasys/teku/dataproviders/generators/CachingTaskQueueTest.java b/ethereum/dataproviders/src/test/java/tech/pegasys/teku/dataproviders/generators/CachingTaskQueueTest.java index 914aaf63591..26d0fee34a0 100644 --- a/ethereum/dataproviders/src/test/java/tech/pegasys/teku/dataproviders/generators/CachingTaskQueueTest.java +++ b/ethereum/dataproviders/src/test/java/tech/pegasys/teku/dataproviders/generators/CachingTaskQueueTest.java @@ -272,33 +272,29 @@ private void assertCacheSizeMetric(final int expectedSize) { private void assertCacheHitCount(final int expectedCount) { final double value = - metricsSystem - .getCounter(TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total") - .getValue("cached"); + metricsSystem.getCounterValue( + TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total", "cached"); assertThat(value).isEqualTo(expectedCount); } private void assertNewTaskCount(final int expectedCount) { final double value = - metricsSystem - .getCounter(TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total") - .getValue("new"); + metricsSystem.getCounterValue( + TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total", "new"); assertThat(value).isEqualTo(expectedCount); } private void assertDuplicateTaskCount(final int expectedCount) { final double value = - metricsSystem - .getCounter(TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total") - .getValue("duplicate"); + metricsSystem.getCounterValue( + TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total", "duplicate"); assertThat(value).isEqualTo(expectedCount); } private void assertRebasedTaskCount(final int expectedCount) { final double value = - metricsSystem - .getCounter(TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total") - .getValue("rebase"); + metricsSystem.getCounterValue( + TekuMetricCategory.STORAGE, METRICS_PREFIX + "_tasks_total", "rebase"); assertThat(value).isEqualTo(expectedCount); } diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerBlockProductionManagerImplTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerBlockProductionManagerImplTest.java index 6059e89dab2..cb24954d9e4 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerBlockProductionManagerImplTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerBlockProductionManagerImplTest.java @@ -567,9 +567,11 @@ private void verifyEngineCalled( private void verifySourceCounter(final Source source, final FallbackReason reason) { final long actualCount = - stubMetricsSystem - .getCounter(TekuMetricCategory.BEACON, "execution_payload_source_total") - .getValue(source.toString(), reason.toString()); + stubMetricsSystem.getCounterValue( + TekuMetricCategory.BEACON, + "execution_payload_source_total", + source.toString(), + reason.toString()); assertThat(actualCount).isOne(); } } diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java index 6dd9c84b009..c81a5e10c9c 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java @@ -887,9 +887,11 @@ private void verifyEngineCalled( private void verifySourceCounter(final Source source, final FallbackReason reason) { final long actualCount = - stubMetricsSystem - .getCounter(TekuMetricCategory.BEACON, "execution_payload_source_total") - .getValue(source.toString(), reason.toString()); + stubMetricsSystem.getCounterValue( + TekuMetricCategory.BEACON, + "execution_payload_source_total", + source.toString(), + reason.toString()); assertThat(actualCount).isOne(); } } diff --git a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/interop/MergedGenesisTestBuilder.java b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/interop/MergedGenesisTestBuilder.java index fc08f3fbabb..77cd30077ca 100644 --- a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/interop/MergedGenesisTestBuilder.java +++ b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/interop/MergedGenesisTestBuilder.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MiningParameters; +import org.hyperledger.besu.ethereum.core.MiningConfiguration; import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -40,7 +40,7 @@ public static ExecutionPayloadHeader createPayloadForBesuGenesis( final ProtocolSchedule protocolSchedule = MainnetProtocolSchedule.fromConfig( genesisConfigOptions, - MiningParameters.MINING_DISABLED, + MiningConfiguration.MINING_DISABLED, badBlockManager, false, new NoOpMetricsSystem()); diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ProposersDataManager.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ProposersDataManager.java index ed130f50c45..fddc3935024 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ProposersDataManager.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ProposersDataManager.java @@ -23,7 +23,7 @@ import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; import tech.pegasys.teku.ethereum.events.SlotEventsChannel; import tech.pegasys.teku.ethereum.execution.types.Eth1Address; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -67,10 +67,10 @@ public ProposersDataManager( final RecentChainData recentChainData, final Optional proposerDefaultFeeRecipient, final boolean forkChoiceUpdatedAlwaysSendPayloadAttribute) { - final LabelledGauge labelledGauge = - metricsSystem.createLabelledGauge( + final LabelledSuppliedMetric labelledGauge = + metricsSystem.createLabelledSuppliedGauge( TekuMetricCategory.BEACON, - "proposers_data_total", + "proposers_data", "Total number proposers/validators under management", "type"); diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/signatures/AggregatingSignatureVerificationService.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/signatures/AggregatingSignatureVerificationService.java index 15f3597a76e..4887ecbfe2b 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/signatures/AggregatingSignatureVerificationService.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/signatures/AggregatingSignatureVerificationService.java @@ -27,6 +27,7 @@ import org.apache.tuweni.bytes.Bytes; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.Histogram; import tech.pegasys.teku.bls.BLS; import tech.pegasys.teku.bls.BLSPublicKey; import tech.pegasys.teku.bls.BLSSignature; @@ -53,7 +54,7 @@ public class AggregatingSignatureVerificationService extends SignatureVerificati private final AsyncRunner asyncRunner; private final Counter batchCounter; private final Counter taskCounter; - private final MetricsHistogram batchSizeHistogram; + private final Histogram batchSizeHistogram; @VisibleForTesting AggregatingSignatureVerificationService( @@ -89,13 +90,11 @@ public class AggregatingSignatureVerificationService extends SignatureVerificati "signature_verifications_task_count_total", "Reports the number of individual verification tasks processed"); batchSizeHistogram = - MetricsHistogram.create( + metricsSystem.createHistogram( TekuMetricCategory.EXECUTOR, - metricsSystem, "signature_verifications_batch_size", "Histogram of signature verification batch sizes", - 3, - List.of()); + MetricsHistogram.getDefaultBuckets()); } public AggregatingSignatureVerificationService( @@ -188,7 +187,7 @@ private List waitForBatch() { void batchVerifySignatures(final List tasks) { batchCounter.inc(); taskCounter.inc(tasks.size()); - batchSizeHistogram.recordValue(tasks.size()); + batchSizeHistogram.observe(tasks.size()); final List> allKeys = new ArrayList<>(); final List allMessages = new ArrayList<>(); final List allSignatures = new ArrayList<>(); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ProposerDataManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ProposerDataManagerTest.java index 5130bd2bcd9..ced82c559c9 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ProposerDataManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ProposerDataManagerTest.java @@ -155,7 +155,7 @@ private void prepareRegistrations() { private void assertPreparedProposersCount(final int expectedCount) { final OptionalDouble optionalValue = metricsSystem - .getLabelledGauge(TekuMetricCategory.BEACON, "proposers_data_total") + .getLabelledGauge(TekuMetricCategory.BEACON, "proposers_data") .getValue("prepared_proposers"); assertThat(optionalValue).hasValue(expectedCount); } @@ -163,7 +163,7 @@ private void assertPreparedProposersCount(final int expectedCount) { private void assertRegisteredValidatorsCount(final int expectedCount) { final OptionalDouble optionalValue = metricsSystem - .getLabelledGauge(TekuMetricCategory.BEACON, "proposers_data_total") + .getLabelledGauge(TekuMetricCategory.BEACON, "proposers_data") .getValue("registered_validators"); assertThat(optionalValue).hasValue(expectedCount); } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java index 2dad121a9eb..d7998302d19 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java @@ -41,6 +41,7 @@ import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.Observation; import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import tech.pegasys.infrastructure.logging.LogCaptor; @@ -78,6 +79,7 @@ public class BlockBlobSidecarsTrackersPoolImplTest { private final StubAsyncRunner asyncRunner = new StubAsyncRunner(); private final RecentChainData recentChainData = mock(RecentChainData.class); private final ExecutionLayerChannel executionLayer = mock(ExecutionLayerChannel.class); + BlockBlobSidecarsTrackersPoolImpl blockBlobSidecarsTrackersPool; @SuppressWarnings("unchecked") private final Function> blobSidecarPublisher = mock(Function.class); @@ -87,21 +89,6 @@ public class BlockBlobSidecarsTrackersPoolImplTest { private final BlockImportChannel blockImportChannel = mock(BlockImportChannel.class); private final int maxItems = 15; - private final BlockBlobSidecarsTrackersPoolImpl blockBlobSidecarsTrackersPool = - new PoolFactory(metricsSystem) - .createPoolForBlockBlobSidecarsTrackers( - blockImportChannel, - spec, - timeProvider, - asyncRunner, - recentChainData, - executionLayer, - () -> blobSidecarGossipValidator, - blobSidecarPublisher, - historicalTolerance, - futureTolerance, - maxItems, - this::trackerFactory); private UInt64 currentSlot = historicalTolerance.times(2); private final List requiredBlockRootEvents = new ArrayList<>(); @@ -115,6 +102,21 @@ public class BlockBlobSidecarsTrackersPoolImplTest { @BeforeEach public void setup() { + blockBlobSidecarsTrackersPool = + new PoolFactory(metricsSystem) + .createPoolForBlockBlobSidecarsTrackers( + blockImportChannel, + spec, + timeProvider, + asyncRunner, + recentChainData, + executionLayer, + () -> blobSidecarGossipValidator, + blobSidecarPublisher, + historicalTolerance, + futureTolerance, + maxItems, + this::trackerFactory); // Set up slot blockBlobSidecarsTrackersPool.subscribeRequiredBlockRoot(requiredBlockRootEvents::add); blockBlobSidecarsTrackersPool.subscribeRequiredBlockRootDropped( @@ -127,6 +129,11 @@ public void setup() { setSlot(currentSlot); } + @AfterEach + public void tearDown() { + metricsSystem.shutdown(); + } + private void setSlot(final long slot) { setSlot(UInt64.valueOf(slot)); } @@ -1369,7 +1376,7 @@ private static BlobIdentifier blobIdentifierFromBlobSidecar(final BlobSidecar bl private void assertStats(final String type, final String subType, final double count) { assertThat( - getMetricsValues("block_blobs_trackers_pool_stats_total").get(List.of(type, subType))) + getMetricsValues("block_blobs_trackers_pool_stats_total").get(List.of(subType, type))) .isEqualTo(count); } @@ -1392,8 +1399,8 @@ private void assertBlobSidecarsTrackersCount(final int count) { private Map, Object> getMetricsValues(final String metricName) { return metricsSystem .streamObservations(TekuMetricCategory.BEACON) - .filter(ob -> ob.getMetricName().equals(metricName)) - .collect(Collectors.toMap(Observation::getLabels, Observation::getValue)); + .filter(ob -> ob.metricName().equals(metricName)) + .collect(Collectors.toMap(Observation::labels, Observation::value)); } private BlockBlobSidecarsTracker trackerFactory(final SlotAndBlockRoot slotAndBlockRoot) { diff --git a/gradle/versions.gradle b/gradle/versions.gradle index b49b0042dde..09dbc7f7be8 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -139,14 +139,14 @@ dependencyManagement { dependency 'org.xerial.snappy:snappy-java:1.1.10.7' - dependency 'io.prometheus:simpleclient:0.16.0' + dependency 'io.prometheus:prometheus-metrics-bom:1.3.4' - dependencySet(group: 'org.hyperledger.besu.internal', version: '24.10.0') { + dependencySet(group: 'org.hyperledger.besu.internal', version: '24.12.0') { entry('metrics-core') entry('core') entry('config') } - dependencySet(group: 'org.hyperledger.besu', version: '24.10.0') { + dependencySet(group: 'org.hyperledger.besu', version: '24.12.0') { entry('besu-datatypes') entry('evm') entry('plugin-api') diff --git a/infrastructure/async/src/main/java/tech/pegasys/teku/infrastructure/async/MetricTrackingExecutorFactory.java b/infrastructure/async/src/main/java/tech/pegasys/teku/infrastructure/async/MetricTrackingExecutorFactory.java index bc9c0def2e2..64aebc0597a 100644 --- a/infrastructure/async/src/main/java/tech/pegasys/teku/infrastructure/async/MetricTrackingExecutorFactory.java +++ b/infrastructure/async/src/main/java/tech/pegasys/teku/infrastructure/async/MetricTrackingExecutorFactory.java @@ -21,14 +21,14 @@ import java.util.concurrent.TimeUnit; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; -import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; public class MetricTrackingExecutorFactory { private final MetricsSystem metricsSystem; - private final LabelledGauge labelledGaugeQueueSize; + private final LabelledSuppliedMetric labelledGaugeQueueSize; private final LabelledMetric labelledGaugeRejectedExecutions; private final OccurrenceCounter rejectedExecutionCounter; @@ -41,7 +41,7 @@ public MetricTrackingExecutorFactory( final MetricsSystem metricsSystem, final OccurrenceCounter rejectedExecutionCounter) { this.metricsSystem = metricsSystem; this.labelledGaugeQueueSize = - metricsSystem.createLabelledGauge( + metricsSystem.createLabelledSuppliedGauge( TekuMetricCategory.EXECUTOR, "queue_size", "Current size of the executor task queue", diff --git a/infrastructure/async/src/main/java/tech/pegasys/teku/infrastructure/async/ThrottlingTaskQueueWithPriority.java b/infrastructure/async/src/main/java/tech/pegasys/teku/infrastructure/async/ThrottlingTaskQueueWithPriority.java index bca0afc4650..c1be6686f87 100644 --- a/infrastructure/async/src/main/java/tech/pegasys/teku/infrastructure/async/ThrottlingTaskQueueWithPriority.java +++ b/infrastructure/async/src/main/java/tech/pegasys/teku/infrastructure/async/ThrottlingTaskQueueWithPriority.java @@ -17,7 +17,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.Supplier; import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; public class ThrottlingTaskQueueWithPriority extends ThrottlingTaskQueue { @@ -31,8 +31,8 @@ public static ThrottlingTaskQueueWithPriority create( final String metricName) { final ThrottlingTaskQueueWithPriority taskQueue = new ThrottlingTaskQueueWithPriority(maximumConcurrentTasks); - final LabelledGauge taskQueueGauge = - metricsSystem.createLabelledGauge( + final LabelledSuppliedMetric taskQueueGauge = + metricsSystem.createLabelledSuppliedGauge( metricCategory, metricName, "Number of tasks queued", "priority"); taskQueueGauge.labels(taskQueue.queuedTasks::size, "normal"); taskQueueGauge.labels(taskQueue.queuedPrioritizedTasks::size, "high"); diff --git a/infrastructure/metrics/build.gradle b/infrastructure/metrics/build.gradle index f300d1146e7..d5c7049fc74 100644 --- a/infrastructure/metrics/build.gradle +++ b/infrastructure/metrics/build.gradle @@ -5,12 +5,13 @@ dependencies { implementation project(':infrastructure:io') implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'io.prometheus:simpleclient' + implementation 'io.prometheus:prometheus-metrics-bom' implementation 'io.vertx:vertx-core' implementation 'io.vertx:vertx-web' implementation 'io.tmio:tuweni-units' implementation 'org.hdrhistogram:HdrHistogram' implementation 'org.hyperledger.besu.internal:metrics-core' + testFixturesApi 'com.google.guava:guava' testFixturesApi 'org.hyperledger.besu:plugin-api' } diff --git a/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/MetricsEndpoint.java b/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/MetricsEndpoint.java index f5b1eaf75e6..ff4a3a14a54 100644 --- a/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/MetricsEndpoint.java +++ b/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/MetricsEndpoint.java @@ -13,7 +13,6 @@ package tech.pegasys.teku.infrastructure.metrics; -import io.vertx.core.Vertx; import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.hyperledger.besu.metrics.MetricsService; @@ -27,11 +26,11 @@ public class MetricsEndpoint { private final PrometheusMetricsSystem metricsSystem; private final MetricsConfig config; - public MetricsEndpoint(final MetricsConfig config, final Vertx vertx) { + public MetricsEndpoint(final MetricsConfig config) { final MetricsConfiguration metricsConfig = createMetricsConfiguration(config); metricsSystem = new PrometheusMetricsSystem(config.getMetricsCategories(), true); metricsSystem.init(); - metricsService = MetricsService.create(vertx, metricsConfig, metricsSystem); + metricsService = MetricsService.create(metricsConfig, metricsSystem); this.config = config; } diff --git a/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/MetricsHistogram.java b/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/MetricsHistogram.java index 56719ff13cf..1662c06fabd 100644 --- a/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/MetricsHistogram.java +++ b/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/MetricsHistogram.java @@ -13,21 +13,7 @@ package tech.pegasys.teku.infrastructure.metrics; -import static com.google.common.base.Preconditions.checkArgument; - -import io.prometheus.client.Collector; -import io.prometheus.client.Collector.MetricFamilySamples; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Stream; -import org.HdrHistogram.SynchronizedHistogram; import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; -import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.MetricCategory; /** * A histogram metric that automatically selects bucket sizes based on simple configuration and the @@ -39,185 +25,16 @@ * @see HdrHistogram docs */ public class MetricsHistogram { - static final String QUANTILE_LABEL = "quantile"; - static final String LABEL_50 = "0.5"; - static final String LABEL_95 = "0.95"; - static final String LABEL_99 = "0.99"; - static final String LABEL_1 = "1"; - - private final Map, SynchronizedHistogram> histogramMap = new ConcurrentHashMap<>(); - private final List labels; - private final Optional highestTrackableValue; - private final int numberOfSignificantValueDigits; - - protected MetricsHistogram( - final int numberOfSignificantValueDigits, - final Optional highestTrackableValue, - final List customLabelsNames) { - this.numberOfSignificantValueDigits = numberOfSignificantValueDigits; - this.highestTrackableValue = highestTrackableValue; - this.labels = Stream.concat(customLabelsNames.stream(), Stream.of(QUANTILE_LABEL)).toList(); - } - - /** - * Create a new histogram metric which auto-resizes to fit any values supplied and maintains at - * least {@code numberOfSignificantValueDigits} of precision. - * - * @param category the metrics category - * @param metricsSystem the metrics system to register with - * @param name the name of the metric - * @param help the help text describing the metric - * @param numberOfSignificantValueDigits the number of digits of precision to preserve - * @return the new metric - */ - public static MetricsHistogram create( - final MetricCategory category, - final MetricsSystem metricsSystem, - final String name, - final String help, - final int numberOfSignificantValueDigits, - final List customLabelsNames) { - - return createMetric( - category, - metricsSystem, - name, - help, - numberOfSignificantValueDigits, - Optional.empty(), - customLabelsNames); - } - - /** - * Create a new histogram metric with a fixed maximum value, maintaining at least {@code - * numberOfSignificantValueDigits} of precision. - * - *

Values above the specified highestTrackableValue will be recorded as being equal to that - * value. - * - * @param category the metrics category - * @param metricsSystem the metrics system to register with - * @param name the name of the metric - * @param help the help text describing the metric - * @param numberOfSignificantValueDigits the number of digits of precision to preserve - * @param highestTrackableValue the highest value that can be recorded by this histogram - * @return the new metric - */ - public static MetricsHistogram create( - final MetricCategory category, - final MetricsSystem metricsSystem, - final String name, - final String help, - final int numberOfSignificantValueDigits, - final long highestTrackableValue, - final List customLabelsNames) { - - return createMetric( - category, - metricsSystem, - name, - help, - numberOfSignificantValueDigits, - Optional.of(highestTrackableValue), - customLabelsNames); - } - private static MetricsHistogram createMetric( - final MetricCategory category, - final MetricsSystem metricsSystem, - final String name, - final String help, - final int numberOfSignificantValueDigits, - final Optional highestTrackableValue, - final List customLabelsNames) { + static final double LABEL_0 = 0.0; + static final double LABEL_50 = 0.5; + static final double LABEL_95 = 0.95; + static final double LABEL_99 = 0.99; + static final double LABEL_1 = 1; - final MetricsHistogram histogram = - new MetricsHistogram( - numberOfSignificantValueDigits, highestTrackableValue, customLabelsNames); - if (metricsSystem instanceof PrometheusMetricsSystem) { - ((PrometheusMetricsSystem) metricsSystem) - .addCollector(category, () -> histogram.histogramToCollector(category, name, help)); - } - return histogram; - } - - public void recordValue(final long value, final String... customLabelValues) { - checkArgument( - labels.size() == customLabelValues.length + 1, - "customLabelsNames and customLabelsValues must have the same size"); - - final SynchronizedHistogram histogram = - histogramMap.computeIfAbsent( - Arrays.asList(customLabelValues), - __ -> - highestTrackableValue - .map(aLong -> new SynchronizedHistogram(aLong, numberOfSignificantValueDigits)) - .orElseGet(() -> new SynchronizedHistogram(numberOfSignificantValueDigits))); - - if (histogram.isAutoResize()) { - histogram.recordValue(value); - } else { - histogram.recordValue(Math.min(histogram.getHighestTrackableValue(), value)); - } - } - - protected Collector histogramToCollector( - final MetricCategory metricCategory, final String name, final String help) { - return new Collector() { - final String metricName = - metricCategory.getApplicationPrefix().orElse("") + metricCategory.getName() + "_" + name; - - @Override - public List collect() { - - final List samples = - histogramMap.entrySet().stream() - .map( - labelsValuesToHistogram -> - List.of( - createSample( - metricName, - LABEL_50, - labelsValuesToHistogram.getKey(), - 50d, - labelsValuesToHistogram.getValue()), - createSample( - metricName, - LABEL_95, - labelsValuesToHistogram.getKey(), - 95d, - labelsValuesToHistogram.getValue()), - createSample( - metricName, - LABEL_99, - labelsValuesToHistogram.getKey(), - 99d, - labelsValuesToHistogram.getValue()), - createSample( - metricName, - LABEL_1, - labelsValuesToHistogram.getKey(), - labelsValuesToHistogram.getValue().getMaxValueAsDouble(), - labelsValuesToHistogram.getValue()))) - .flatMap(List::stream) - .toList(); - - return Collections.singletonList( - new MetricFamilySamples(metricName, Type.SUMMARY, help, samples)); - } - }; - } + static final double[] DEFAULT_BUCKETS = {LABEL_0, LABEL_50, LABEL_95, LABEL_99, LABEL_1}; - private MetricFamilySamples.Sample createSample( - final String metricName, - final String quantileLabelValue, - final List labelValues, - final double percentile, - final SynchronizedHistogram histogram) { - return new MetricFamilySamples.Sample( - metricName, - labels, - Stream.concat(labelValues.stream(), Stream.of(quantileLabelValue)).toList(), - histogram.getValueAtPercentile(percentile)); + public static double[] getDefaultBuckets() { + return DEFAULT_BUCKETS; } } diff --git a/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/SettableLabelledGauge.java b/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/SettableLabelledGauge.java index 6eb0855f936..5bf207828c0 100644 --- a/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/SettableLabelledGauge.java +++ b/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/SettableLabelledGauge.java @@ -18,15 +18,15 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; public class SettableLabelledGauge { private final Map, AtomicDouble> valueHolders = new ConcurrentHashMap<>(); - private final LabelledGauge labelledGauge; + private final LabelledSuppliedMetric labelledGauge; - private SettableLabelledGauge(final LabelledGauge labelledGauge) { + private SettableLabelledGauge(final LabelledSuppliedMetric labelledGauge) { this.labelledGauge = labelledGauge; } @@ -36,8 +36,8 @@ public static SettableLabelledGauge create( final String name, final String help, final String... labels) { - final LabelledGauge labelledGauge = - metricsSystem.createLabelledGauge(category, name, help, labels); + final LabelledSuppliedMetric labelledGauge = + metricsSystem.createLabelledSuppliedGauge(category, name, help, labels); return new SettableLabelledGauge(labelledGauge); } diff --git a/infrastructure/metrics/src/test/java/tech/pegasys/teku/infrastructure/metrics/MetricsCountersByIntervalsTest.java b/infrastructure/metrics/src/test/java/tech/pegasys/teku/infrastructure/metrics/MetricsCountersByIntervalsTest.java index 8b8615d7847..904224da424 100644 --- a/infrastructure/metrics/src/test/java/tech/pegasys/teku/infrastructure/metrics/MetricsCountersByIntervalsTest.java +++ b/infrastructure/metrics/src/test/java/tech/pegasys/teku/infrastructure/metrics/MetricsCountersByIntervalsTest.java @@ -23,15 +23,26 @@ import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.Observation; import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; public class MetricsCountersByIntervalsTest { private static final TekuMetricCategory CATEGORY = TekuMetricCategory.BEACON; - private final ObservableMetricsSystem metricsSystem = - new PrometheusMetricsSystem(Set.of(CATEGORY), true); - private static final String COUNTER_NAME = "metric_counter"; + private ObservableMetricsSystem metricsSystem; + + @BeforeEach + void setup() { + metricsSystem = new PrometheusMetricsSystem(Set.of(CATEGORY), true); + } + + @AfterEach + void cleanup() { + metricsSystem.shutdown(); + } + @Test void shouldCountWithDefault() { final Map, List> eventsAndBoundaries = @@ -72,27 +83,27 @@ void shouldCountWithDefault() { final Map, Object> values = metricsSystem .streamObservations(CATEGORY) - .filter(ob -> ob.getMetricName().equals(COUNTER_NAME)) - .filter(ob -> !ob.getLabels().contains("created")) - .collect(Collectors.toMap(Observation::getLabels, Observation::getValue)); + .filter(ob -> ob.metricName().equals(COUNTER_NAME)) + .filter(ob -> !ob.labels().contains("created")) + .collect(Collectors.toMap(Observation::labels, Observation::value)); assertThat(values) .containsOnly( // fallback - entry(List.of("unknownLabelVal", "unknownLabelVal", "[0,∞)"), 100d), + entry(List.of("[0,∞)", "unknownLabelVal", "unknownLabelVal"), 100d), // first rule - entry(List.of("label1Val1", "label2UnknownVal", "[0,10)"), 9d), - entry(List.of("label1Val1", "label2UnknownVal", "[10,50)"), 40d), - entry(List.of("label1Val1", "label2UnknownVal", "[50,80)"), 30d), - entry(List.of("label1Val1", "label2UnknownVal", "[80,∞)"), 21d), + entry(List.of("[0,10)", "label1Val1", "label2UnknownVal"), 9d), + entry(List.of("[10,50)", "label1Val1", "label2UnknownVal"), 40d), + entry(List.of("[50,80)", "label1Val1", "label2UnknownVal"), 30d), + entry(List.of("[80,∞)", "label1Val1", "label2UnknownVal"), 21d), // second rule - entry(List.of("label1Val2", "label2UnknownVal", "[0,50)"), 49d), - entry(List.of("label1Val2", "label2UnknownVal", "[50,∞)"), 51d), + entry(List.of("[0,50)", "label1Val2", "label2UnknownVal"), 49d), + entry(List.of("[50,∞)", "label1Val2", "label2UnknownVal"), 51d), // third rule - entry(List.of("label1Val2", "label2Val", "[0,80)"), 79d), - entry(List.of("label1Val2", "label2Val", "[80,∞)"), 21d)); + entry(List.of("[0,80)", "label1Val2", "label2Val"), 79d), + entry(List.of("[80,∞)", "label1Val2", "label2Val"), 21d)); } @Test @@ -117,11 +128,11 @@ void shouldNotCountNonMatching() { final Map, Object> values = metricsSystem .streamObservations(CATEGORY) - .filter(ob -> ob.getMetricName().equals(COUNTER_NAME)) - .filter(ob -> !ob.getLabels().contains("created")) - .collect(Collectors.toMap(Observation::getLabels, Observation::getValue)); + .filter(ob -> ob.metricName().equals(COUNTER_NAME)) + .filter(ob -> !ob.labels().contains("created")) + .collect(Collectors.toMap(Observation::labels, Observation::value)); - assertThat(values).containsOnly(entry(List.of("label1Val1", "label2UnknownVal", "[0,∞)"), 1d)); + assertThat(values).containsOnly(entry(List.of("[0,∞)", "label1Val1", "label2UnknownVal"), 1d)); } @Test @@ -142,10 +153,10 @@ void shouldInitCounters() { final Map, Object> values = metricsSystem .streamObservations(CATEGORY) - .filter(ob -> ob.getMetricName().equals(COUNTER_NAME)) - .filter(ob -> !ob.getLabels().contains("created")) - .collect(Collectors.toMap(Observation::getLabels, Observation::getValue)); + .filter(ob -> ob.metricName().equals(COUNTER_NAME)) + .filter(ob -> !ob.labels().contains("created")) + .collect(Collectors.toMap(Observation::labels, Observation::value)); - assertThat(values).containsOnly(entry(List.of("a", "b", "[0,∞)"), 0d)); + assertThat(values).containsOnly(entry(List.of("[0,∞)", "a", "b"), 0d)); } } diff --git a/infrastructure/metrics/src/test/java/tech/pegasys/teku/infrastructure/metrics/MetricsHistogramTest.java b/infrastructure/metrics/src/test/java/tech/pegasys/teku/infrastructure/metrics/MetricsHistogramTest.java deleted file mode 100644 index f32f1b60a0c..00000000000 --- a/infrastructure/metrics/src/test/java/tech/pegasys/teku/infrastructure/metrics/MetricsHistogramTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.metrics; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.metrics.Observation; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; -import org.junit.jupiter.api.Test; - -class MetricsHistogramTest { - - private static final TekuMetricCategory CATEGORY = TekuMetricCategory.BEACON; - private final ObservableMetricsSystem metricsSystem = - new PrometheusMetricsSystem(Set.of(CATEGORY), true); - - @Test - void shouldReportValuesWithNoSpecifiedUpperLimit() { - final MetricsHistogram histogram = - MetricsHistogram.create(CATEGORY, metricsSystem, "test", "Test help", 3, List.of()); - - for (int i = 1; i <= 100; i++) { - histogram.recordValue(i); - } - final Map, Object> values = - metricsSystem - .streamObservations() - .filter(ob -> ob.getCategory() == CATEGORY) - .collect(Collectors.toMap(Observation::getLabels, Observation::getValue)); - assertThat(values) - .containsOnly( - entry(key(List.of(MetricsHistogram.LABEL_50)), 50d), - entry(key(List.of(MetricsHistogram.LABEL_95)), 95d), - entry(key(List.of(MetricsHistogram.LABEL_99)), 99d), - entry(key(List.of(MetricsHistogram.LABEL_1)), 100d)); - } - - @Test - void shouldReportValuesWithUpperLimit() { - final MetricsHistogram histogram = - MetricsHistogram.create(CATEGORY, metricsSystem, "test", "Test help", 2, 80, List.of()); - - for (int i = 1; i <= 100; i++) { - histogram.recordValue(i); - } - final Map, Object> values = - metricsSystem - .streamObservations() - .filter(ob -> ob.getCategory() == CATEGORY) - .collect(Collectors.toMap(Observation::getLabels, Observation::getValue)); - assertThat(values) - .containsOnly( - entry(key(List.of(MetricsHistogram.LABEL_50)), 50d), - entry(key(List.of(MetricsHistogram.LABEL_95)), 80d), - entry(key(List.of(MetricsHistogram.LABEL_99)), 80d), - entry(key(List.of(MetricsHistogram.LABEL_1)), 80d)); - } - - private static List key(final List labelValues) { - final List key = new ArrayList<>(); - key.add(MetricsHistogram.QUANTILE_LABEL); - key.addAll(labelValues); - return key; - } -} diff --git a/infrastructure/metrics/src/testFixtures/java/tech/pegasys/teku/infrastructure/metrics/StubLabelledGauge.java b/infrastructure/metrics/src/testFixtures/java/tech/pegasys/teku/infrastructure/metrics/StubLabelledGauge.java index 0bb4f5f4311..2d7d3e14ed4 100644 --- a/infrastructure/metrics/src/testFixtures/java/tech/pegasys/teku/infrastructure/metrics/StubLabelledGauge.java +++ b/infrastructure/metrics/src/testFixtures/java/tech/pegasys/teku/infrastructure/metrics/StubLabelledGauge.java @@ -18,10 +18,10 @@ import java.util.OptionalDouble; import java.util.concurrent.ConcurrentHashMap; import java.util.function.DoubleSupplier; -import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; -public class StubLabelledGauge extends StubMetric implements LabelledGauge { +public class StubLabelledGauge extends StubMetric implements LabelledSuppliedMetric { private final Map, DoubleSupplier> suppliers = new ConcurrentHashMap<>(); diff --git a/infrastructure/metrics/src/testFixtures/java/tech/pegasys/teku/infrastructure/metrics/StubMetricsSystem.java b/infrastructure/metrics/src/testFixtures/java/tech/pegasys/teku/infrastructure/metrics/StubMetricsSystem.java index 0016785f19d..f07d3f2e629 100644 --- a/infrastructure/metrics/src/testFixtures/java/tech/pegasys/teku/infrastructure/metrics/StubMetricsSystem.java +++ b/infrastructure/metrics/src/testFixtures/java/tech/pegasys/teku/infrastructure/metrics/StubMetricsSystem.java @@ -13,21 +13,35 @@ package tech.pegasys.teku.infrastructure.metrics; +import static java.util.Arrays.asList; + +import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.DoubleSupplier; +import java.util.function.Supplier; import java.util.regex.Pattern; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; +import org.hyperledger.besu.plugin.services.metrics.Histogram; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedSummary; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.hyperledger.besu.plugin.services.metrics.OperationTimer; public class StubMetricsSystem implements MetricsSystem { - private final Map> counters = new ConcurrentHashMap<>(); + private final Map> counters = + new ConcurrentHashMap<>(); private final Map> gauges = new ConcurrentHashMap<>(); private final Map> labelledGauges = new ConcurrentHashMap<>(); @@ -37,6 +51,9 @@ public class StubMetricsSystem implements MetricsSystem { private static final Pattern METRIC_NAME_PATTERN = Pattern.compile("[a-zA-Z_:][a-zA-Z0-9_:]*"); private static final Pattern LABEL_NAME_PATTERN = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*"); + /** The constant NO_OP_HISTOGRAM. */ + public static final Histogram NO_OP_HISTOGRAM = d -> {}; + @Override public LabelledMetric createLabelledCounter( final MetricCategory category, @@ -47,11 +64,20 @@ public LabelledMetric createLabelledCounter( validateLabelName(labelNames); return counters .computeIfAbsent(category, __ -> new ConcurrentHashMap<>()) - .computeIfAbsent(name, __ -> new StubCounter()); + .computeIfAbsent(name, __ -> new StubLabelledCounter()); + } + + @Override + public LabelledSuppliedMetric createLabelledSuppliedCounter( + final MetricCategory metricCategory, + final String s, + final String s1, + final String... strings) { + return null; } @Override - public LabelledGauge createLabelledGauge( + public LabelledSuppliedMetric createLabelledSuppliedGauge( final MetricCategory category, final String name, final String help, @@ -63,6 +89,20 @@ public LabelledGauge createLabelledGauge( .computeIfAbsent(name, __ -> new StubLabelledGauge(category, name, help)); } + public long getCounterValue( + final MetricCategory category, final String name, final String... labels) { + final Map counters = this.counters.get(category); + final StubLabelledCounter labelledCounter = counters.get(name); + if (labelledCounter == null) { + throw new IllegalArgumentException("Unknown counter: " + name); + } + final StubCounter metric = labelledCounter.getMetric(labels); + if (metric == null) { + return 0; + } + return metric.getValue(); + } + @Override public void createGauge( final MetricCategory category, @@ -79,6 +119,34 @@ public void createGauge( } } + @Override + public LabelledMetric createLabelledHistogram( + final MetricCategory category, + final String name, + final String help, + final double[] buckets, + final String... labelNames) { + return new LabelCountingNoOpMetric<>(labelNames.length, NO_OP_HISTOGRAM); + } + + @Override + public LabelledSuppliedSummary createLabelledSuppliedSummary( + final MetricCategory category, + final String name, + final String help, + final String... labelNames) { + return new LabelledSuppliedNoOpMetric(labelNames.length); + } + + @Override + public Set getEnabledCategories() { + return Set.of(); + } + + @Override + public void createGuavaCacheCollector( + final MetricCategory category, final String name, final Cache cache) {} + @Override public LabelledMetric createLabelledTimer( final MetricCategory category, @@ -116,7 +184,7 @@ public StubLabelledGauge getLabelledGauge(final MetricCategory category, final S () -> new IllegalArgumentException("Unknown labelled gauge: " + category + " " + name)); } - public StubCounter getCounter(final MetricCategory category, final String name) { + public StubLabelledCounter getCounter(final MetricCategory category, final String name) { validateMetricName(name); return Optional.ofNullable(counters.get(category)) .map(categoryCounters -> categoryCounters.get(name)) @@ -152,4 +220,105 @@ private void validateLabelName(final String... labelNames) { } } } + + public static class LabelCountingNoOpMetric implements LabelledMetric { + + /** The Label count. */ + final int labelCount; + + /** The Fake metric. */ + final T fakeMetric; + + /** + * Instantiates a new Label counting NoOp metric. + * + * @param labelCount the label count + * @param fakeMetric the fake metric + */ + LabelCountingNoOpMetric(final int labelCount, final T fakeMetric) { + this.labelCount = labelCount; + this.fakeMetric = fakeMetric; + } + + @Override + public T labels(final String... labels) { + Preconditions.checkArgument( + labels.length == labelCount, + "The count of labels used must match the count of labels expected."); + return fakeMetric; + } + } + + @SuppressWarnings("removal") // remove when deprecated LabelledGauge is removed + public static class LabelledSuppliedNoOpMetric + implements LabelledSuppliedMetric, LabelledGauge, LabelledSuppliedSummary { + /** The Label count. */ + final int labelCount; + + /** The Label values cache. */ + final List labelValuesCache = new ArrayList<>(); + + /** + * Instantiates a new Labelled supplied NoOp metric. + * + * @param labelCount the label count + */ + public LabelledSuppliedNoOpMetric(final int labelCount) { + this.labelCount = labelCount; + } + + @Override + public void labels(final DoubleSupplier valueSupplier, final String... labelValues) { + internalLabels(valueSupplier, labelValues); + } + + @Override + public void labels( + final Supplier summarySupplier, final String... labelValues) { + internalLabels(summarySupplier, labelValues); + } + + private void internalLabels(final Object valueSupplier, final String... labelValues) { + final String labelValuesString = String.join(",", labelValues); + Preconditions.checkArgument( + !labelValuesCache.contains(labelValuesString), + "Received label values that were already in use " + labelValuesString); + Preconditions.checkArgument( + labelValues.length == labelCount, + "The count of labels used must match the count of labels expected."); + Preconditions.checkNotNull(valueSupplier, "No valueSupplier specified"); + labelValuesCache.add(labelValuesString); + } + } + + public static class StubLabelledCounter implements LabelledMetric { + private final Map, StubCounter> metrics = new HashMap<>(); + + @Override + public Counter labels(final String... labels) { + return metrics.computeIfAbsent(asList(labels), key -> new StubCounter()); + } + + private StubCounter getMetric(final String... labels) { + return metrics.get(asList(labels)); + } + } + + public static class StubCounter implements Counter { + private long value = 0; + + @Override + public void inc() { + value++; + } + + @Override + public void inc(final long amount) { + value += amount; + } + + public long getValue() { + return value; + } + } } diff --git a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/AttestationGossipManagerTest.java b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/AttestationGossipManagerTest.java index 94e562b650c..680fdeafe4b 100644 --- a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/AttestationGossipManagerTest.java +++ b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/AttestationGossipManagerTest.java @@ -25,7 +25,6 @@ import org.junit.jupiter.api.Test; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.async.StubAsyncRunner; -import tech.pegasys.teku.infrastructure.metrics.StubCounter; import tech.pegasys.teku.infrastructure.metrics.StubMetricsSystem; import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; import tech.pegasys.teku.infrastructure.unsigned.UInt64; @@ -177,15 +176,13 @@ void onNewAttestation_incrementFailureCount() { } private long getPublishSuccessCounterValue() { - return getPublishCounter().getValue("success"); + return metricsSystem.getCounterValue( + TekuMetricCategory.BEACON, "published_attestation_total", "success"); } private long getPublishFailureCounterValue() { - return getPublishCounter().getValue("failure"); - } - - private StubCounter getPublishCounter() { - return metricsSystem.getCounter(TekuMetricCategory.BEACON, "published_attestation_total"); + return metricsSystem.getCounterValue( + TekuMetricCategory.BEACON, "published_attestation_total", "failure"); } private Integer computeSubnetId(final Attestation attestation) { diff --git a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRangeMessageHandlerTest.java b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRangeMessageHandlerTest.java index 89c11a20986..be33be74933 100644 --- a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRangeMessageHandlerTest.java +++ b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRangeMessageHandlerTest.java @@ -167,9 +167,10 @@ public void validateRequest_shouldRejectRequestWhenCountIsTooBig() { maxRequestBlobSidecars))); final long rateLimitedCount = - metricsSystem - .getCounter(TekuMetricCategory.NETWORK, "rpc_blob_sidecars_by_range_requests_total") - .getValue("count_too_big"); + metricsSystem.getCounterValue( + TekuMetricCategory.NETWORK, + "rpc_blob_sidecars_by_range_requests_total", + "count_too_big"); assertThat(rateLimitedCount).isOne(); } @@ -192,9 +193,10 @@ public void shouldNotSendBlobSidecarsIfPeerIsRateLimited() { verify(peer, never()).adjustBlobSidecarsRequest(any(), anyLong()); final long rateLimitedCount = - metricsSystem - .getCounter(TekuMetricCategory.NETWORK, "rpc_blob_sidecars_by_range_requests_total") - .getValue("rate_limited"); + metricsSystem.getCounterValue( + TekuMetricCategory.NETWORK, + "rpc_blob_sidecars_by_range_requests_total", + "rate_limited"); assertThat(rateLimitedCount).isOne(); diff --git a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootMessageHandlerTest.java b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootMessageHandlerTest.java index 8374d9ab8d6..49f02296420 100644 --- a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootMessageHandlerTest.java +++ b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootMessageHandlerTest.java @@ -170,9 +170,10 @@ public void validateRequest_shouldNotAllowRequestLargerThanMaximumAllowed() { }); final long countTooBigCount = - metricsSystem - .getCounter(TekuMetricCategory.NETWORK, "rpc_blob_sidecars_by_root_requests_total") - .getValue("count_too_big"); + metricsSystem.getCounterValue( + TekuMetricCategory.NETWORK, + "rpc_blob_sidecars_by_root_requests_total", + "count_too_big"); assertThat(countTooBigCount).isOne(); } @@ -194,9 +195,8 @@ public void shouldNotSendBlobSidecarsIfPeerIsRateLimited() { verify(peer, never()).adjustBlobSidecarsRequest(any(), anyLong()); final long rateLimitedCount = - metricsSystem - .getCounter(TekuMetricCategory.NETWORK, "rpc_blob_sidecars_by_root_requests_total") - .getValue("rate_limited"); + metricsSystem.getCounterValue( + TekuMetricCategory.NETWORK, "rpc_blob_sidecars_by_root_requests_total", "rate_limited"); assertThat(rateLimitedCount).isOne(); diff --git a/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/PeerManager.java b/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/PeerManager.java index 47b29394bfa..1dc1d9a08a3 100644 --- a/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/PeerManager.java +++ b/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/PeerManager.java @@ -28,7 +28,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; import org.jetbrains.annotations.NotNull; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; @@ -70,8 +70,8 @@ public PeerManager( this.peerScoreFunction = peerScoreFunction; metricsSystem.createGauge( TekuMetricCategory.LIBP2P, "peers", "Tracks number of libp2p peers", this::getPeerCount); - final LabelledGauge peerClientLabelledGauge = - metricsSystem.createLabelledGauge( + final LabelledSuppliedMetric peerClientLabelledGauge = + metricsSystem.createLabelledSuppliedGauge( TekuMetricCategory.LIBP2P, "connected_peers_current", "The number of clients connected by client type", @@ -81,8 +81,8 @@ public PeerManager( peerClientLabelledGauge.labels(() -> countConnectedPeersOfType(type), type.getDisplayName()); } - final LabelledGauge peerDirectionLabelledGauge = - metricsSystem.createLabelledGauge( + final LabelledSuppliedMetric peerDirectionLabelledGauge = + metricsSystem.createLabelledSuppliedGauge( TekuMetricCategory.LIBP2P, "peers_direction_current", "The number of peers by direction including inbound and outbound", diff --git a/networking/p2p/src/test/java/tech/pegasys/teku/networking/p2p/libp2p/PeerManagerTest.java b/networking/p2p/src/test/java/tech/pegasys/teku/networking/p2p/libp2p/PeerManagerTest.java index 554f3770c0a..7bc54e826d7 100644 --- a/networking/p2p/src/test/java/tech/pegasys/teku/networking/p2p/libp2p/PeerManagerTest.java +++ b/networking/p2p/src/test/java/tech/pegasys/teku/networking/p2p/libp2p/PeerManagerTest.java @@ -33,7 +33,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -81,13 +81,13 @@ public void subscribeConnect_singleListener() { @Test public void shouldCreatePeerTypeMetrics() { final MetricsSystem metricsSystem = mock(MetricsSystem.class); - final LabelledGauge peerClientLabelledGauge = mock(LabelledGauge.class); - final LabelledGauge peerDirectionLabelledGauge = mock(LabelledGauge.class); + final LabelledSuppliedMetric peerClientLabelledGauge = mock(LabelledSuppliedMetric.class); + final LabelledSuppliedMetric peerDirectionLabelledGauge = mock(LabelledSuppliedMetric.class); - when(metricsSystem.createLabelledGauge( + when(metricsSystem.createLabelledSuppliedGauge( eq(TekuMetricCategory.LIBP2P), eq("connected_peers_current"), any(), eq("client"))) .thenReturn(peerClientLabelledGauge); - when(metricsSystem.createLabelledGauge( + when(metricsSystem.createLabelledSuppliedGauge( eq(TekuMetricCategory.LIBP2P), eq("peers_direction_current"), any(), eq("direction"))) .thenReturn(peerDirectionLabelledGauge); new PeerManager( diff --git a/storage/build.gradle b/storage/build.gradle index 2f5febe79ed..45556946b5f 100644 --- a/storage/build.gradle +++ b/storage/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' - implementation 'io.prometheus:simpleclient' + implementation 'io.prometheus:prometheus-metrics-bom' implementation 'io.tmio:tuweni-ssz' implementation 'org.hyperledger.besu.internal:metrics-core' implementation 'org.hyperledger.besu:plugin-api' diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java index 30e7020d60e..3dafcddb63c 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java @@ -22,7 +22,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; import tech.pegasys.teku.infrastructure.async.AsyncRunner; import tech.pegasys.teku.infrastructure.async.Cancellable; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -89,8 +89,8 @@ public BlobSidecarPruner( this.storeNonCanonicalBlobSidecars = storeNonCanonicalBlobSidecars; if (blobSidecarsStorageCountersEnabled) { - LabelledGauge labelledGauge = - metricsSystem.createLabelledGauge( + LabelledSuppliedMetric labelledGauge = + metricsSystem.createLabelledSuppliedGauge( TekuMetricCategory.STORAGE, "blob_sidecars", "Statistics for BlobSidecars stored", diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java index 5909b91a84a..fae6e1739df 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java @@ -13,9 +13,6 @@ package tech.pegasys.teku.storage.server.rocksdb; -import io.prometheus.client.Collector; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.function.Supplier; @@ -23,6 +20,7 @@ import org.apache.logging.log4j.Logger; import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.rocksdb.HistogramData; import org.rocksdb.HistogramType; @@ -38,11 +36,6 @@ public class RocksDbStats implements AutoCloseable { private static final Logger LOG = LogManager.getLogger(); - static final List LABELS = Collections.singletonList("quantile"); - static final List LABEL_50 = Collections.singletonList("0.5"); - static final List LABEL_95 = Collections.singletonList("0.95"); - static final List LABEL_99 = Collections.singletonList("0.99"); - // Tickers - RocksDB equivalent of counters static final TickerType[] TICKERS = { TickerType.BLOCK_CACHE_ADD, @@ -195,12 +188,31 @@ public void registerMetrics(final RocksDB database) { if (metricsSystem instanceof PrometheusMetricsSystem) { for (final HistogramType histogram : HISTOGRAMS) { - ((PrometheusMetricsSystem) metricsSystem) - .addCollector(category, () -> histogramToCollector(category, stats, histogram)); + metricsSystem.createSummary( + category, + histogram.name().toLowerCase(Locale.ROOT), + "RocksDB histogram for " + histogram.name(), + () -> provideExternalSummary(stats, histogram)); } } } + private static ExternalSummary provideExternalSummary( + final Statistics stats, final HistogramType histogramType) { + + final HistogramData data = stats.getHistogramData(histogramType); + + return new ExternalSummary( + data.getCount(), + data.getSum(), + List.of( + new ExternalSummary.Quantile(0.0, data.getMin()), + new ExternalSummary.Quantile(0.5, data.getMedian()), + new ExternalSummary.Quantile(0.95, data.getPercentile95()), + new ExternalSummary.Quantile(0.99, data.getPercentile99()), + new ExternalSummary.Quantile(1.0, data.getMax()))); + } + private long getLongProperty(final RocksDB database, final String name) { return ifOpen( () -> { @@ -214,38 +226,6 @@ private long getLongProperty(final RocksDB database, final String name) { 0L); } - private Collector histogramToCollector( - final MetricCategory metricCategory, final Statistics stats, final HistogramType histogram) { - return new Collector() { - final String metricName = - metricCategory.getApplicationPrefix().orElse("") - + metricCategory.getName() - + "_" - + histogram.name().toLowerCase(Locale.ROOT); - - @Override - public List collect() { - return ifOpen( - () -> { - final HistogramData data = stats.getHistogramData(histogram); - return Collections.singletonList( - new MetricFamilySamples( - metricName, - Type.SUMMARY, - "RocksDB histogram for " + metricName, - Arrays.asList( - new MetricFamilySamples.Sample( - metricName, LABELS, LABEL_50, data.getMedian()), - new MetricFamilySamples.Sample( - metricName, LABELS, LABEL_95, data.getPercentile95()), - new MetricFamilySamples.Sample( - metricName, LABELS, LABEL_99, data.getPercentile99())))); - }, - Collections.emptyList()); - } - }; - } - @Override public synchronized void close() { if (closed) { diff --git a/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java b/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java index ae88e4465d1..9dd62c870a6 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java @@ -1092,7 +1092,6 @@ private void disableForkChoicePruneThreshold() { private long getReorgCountMetric(final StorageSystem storageSystem) { return storageSystem .getMetricsSystem() - .getCounter(TekuMetricCategory.BEACON, "reorgs_total") - .getValue(); + .getCounterValue(TekuMetricCategory.BEACON, "reorgs_total"); } } diff --git a/teku/src/main/java/tech/pegasys/teku/AbstractNode.java b/teku/src/main/java/tech/pegasys/teku/AbstractNode.java index cb98c043b0d..37f582ed9c3 100644 --- a/teku/src/main/java/tech/pegasys/teku/AbstractNode.java +++ b/teku/src/main/java/tech/pegasys/teku/AbstractNode.java @@ -95,7 +95,7 @@ protected AbstractNode(final TekuConfiguration tekuConfig) { .build()); reportOverrides(tekuConfig); - this.metricsEndpoint = new MetricsEndpoint(tekuConfig.metricsConfig(), vertx); + this.metricsEndpoint = new MetricsEndpoint(tekuConfig.metricsConfig()); final MetricsSystem metricsSystem = metricsEndpoint.getMetricsSystem(); final TekuDefaultExceptionHandler subscriberExceptionHandler = new TekuDefaultExceptionHandler(); diff --git a/validator/beaconnode/src/test/java/tech/pegasys/teku/validator/beaconnode/metrics/MetricRecordingValidatorApiChannelTest.java b/validator/beaconnode/src/test/java/tech/pegasys/teku/validator/beaconnode/metrics/MetricRecordingValidatorApiChannelTest.java index e72db7fad81..543e251cf09 100644 --- a/validator/beaconnode/src/test/java/tech/pegasys/teku/validator/beaconnode/metrics/MetricRecordingValidatorApiChannelTest.java +++ b/validator/beaconnode/src/test/java/tech/pegasys/teku/validator/beaconnode/metrics/MetricRecordingValidatorApiChannelTest.java @@ -267,10 +267,10 @@ private static Arguments sendDataTest( } private long getCounterValue(final String methodLabel, final RequestOutcome outcome) { - return metricsSystem - .getCounter( - TekuMetricCategory.VALIDATOR, - MetricRecordingValidatorApiChannel.BEACON_NODE_REQUESTS_COUNTER_NAME) - .getValue(methodLabel, outcome.toString()); + return metricsSystem.getCounterValue( + TekuMetricCategory.VALIDATOR, + MetricRecordingValidatorApiChannel.BEACON_NODE_REQUESTS_COUNTER_NAME, + methodLabel, + outcome.toString()); } } diff --git a/validator/client/src/test/java/tech/pegasys/teku/validator/client/PendingDutiesTest.java b/validator/client/src/test/java/tech/pegasys/teku/validator/client/PendingDutiesTest.java index 3617b474885..4208eaff43d 100644 --- a/validator/client/src/test/java/tech/pegasys/teku/validator/client/PendingDutiesTest.java +++ b/validator/client/src/test/java/tech/pegasys/teku/validator/client/PendingDutiesTest.java @@ -31,7 +31,6 @@ import org.junit.jupiter.api.Test; import org.mockito.InOrder; import tech.pegasys.teku.infrastructure.async.SafeFuture; -import tech.pegasys.teku.infrastructure.metrics.StubCounter; import tech.pegasys.teku.infrastructure.metrics.StubMetricsSystem; import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; import tech.pegasys.teku.infrastructure.unsigned.UInt64; @@ -244,9 +243,13 @@ void shouldReportTotalNumberOfSuccessesAndFailuresToMetrics() { } private void validateMetrics(final String duty, final long successCount, final long failCount) { - final StubCounter labelledCounter = - metricsSystem.getCounter(TekuMetricCategory.VALIDATOR, "duties_performed_total"); - assertThat(labelledCounter.getValue(duty, "success")).isEqualTo(successCount); - assertThat(labelledCounter.getValue(duty, "failed")).isEqualTo(failCount); + final long labelledCounterSuccess = + metricsSystem.getCounterValue( + TekuMetricCategory.VALIDATOR, "duties_performed_total", duty, "success"); + final long labelledCounterFailed = + metricsSystem.getCounterValue( + TekuMetricCategory.VALIDATOR, "duties_performed_total", duty, "failed"); + assertThat(labelledCounterSuccess).isEqualTo(successCount); + assertThat(labelledCounterFailed).isEqualTo(failCount); } } diff --git a/validator/remote/src/test/java/tech/pegasys/teku/validator/remote/FailoverValidatorApiHandlerTest.java b/validator/remote/src/test/java/tech/pegasys/teku/validator/remote/FailoverValidatorApiHandlerTest.java index 1ace735c94d..d6863d3645b 100644 --- a/validator/remote/src/test/java/tech/pegasys/teku/validator/remote/FailoverValidatorApiHandlerTest.java +++ b/validator/remote/src/test/java/tech/pegasys/teku/validator/remote/FailoverValidatorApiHandlerTest.java @@ -882,10 +882,11 @@ private long getFailoverCounterValue( final RemoteValidatorApiChannel apiChannel, final String methodLabel, final RequestOutcome outcome) { - return stubMetricsSystem - .getCounter( - TekuMetricCategory.VALIDATOR, - FailoverValidatorApiHandler.REMOTE_BEACON_NODES_REQUESTS_COUNTER_NAME) - .getValue(apiChannel.getEndpoint().toString(), methodLabel, outcome.toString()); + return stubMetricsSystem.getCounterValue( + TekuMetricCategory.VALIDATOR, + FailoverValidatorApiHandler.REMOTE_BEACON_NODES_REQUESTS_COUNTER_NAME, + apiChannel.getEndpoint().toString(), + methodLabel, + outcome.toString()); } } From 4dc4476091531022c0c022d15d9889ae74ac672e Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Tue, 3 Dec 2024 18:52:34 +1000 Subject: [PATCH 2/2] add changelog Signed-off-by: Gabriel Fukushima --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 226335a30bc..600c55af793 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ ## Unreleased Changes ### Breaking Changes +- With the upgrade of the Prometheus Java Metrics library, there are the following changes: + - Gauge names are not allowed to end with `total`, therefore metrics as `beacon_proposers_data_total` and `beacon_eth1_current_period_votes_total` are dropping the `_total` suffix + - The `_created` timestamps are not returned by default. + - Some JVM metrics have changed name to adhere to the OTEL standard (see the table below), [Teku Detailed Dashboard](https://grafana.com/grafana/dashboards/16737-teku-detailed/) is updated to support both names + + | Old Name | New Name | + |---------------------------------|---------------------------------| + | jvm_memory_bytes_committed | jvm_memory_committed_bytes | + | jvm_memory_bytes_init | jvm_memory_init_bytes | + | jvm_memory_bytes_max | jvm_memory_max_bytes | + | jvm_memory_bytes_used | jvm_memory_used_bytes | + | jvm_memory_pool_bytes_committed | jvm_memory_pool_committed_bytes | + | jvm_memory_pool_bytes_init | jvm_memory_pool_init_bytes | + | jvm_memory_pool_bytes_max | jvm_memory_pool_max_bytes | + | jvm_memory_pool_bytes_used | jvm_memory_pool_used_bytes | ### Additions and Improvements - improve block publishing performance, especially relevant with locally produced blocks