diff --git a/core/cmd/shell.go b/core/cmd/shell.go index 788e4da6f69..53ba5e3f82f 100644 --- a/core/cmd/shell.go +++ b/core/cmd/shell.go @@ -53,6 +53,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/cache" "github.com/smartcontractkit/chainlink/v2/core/services/versioning" "github.com/smartcontractkit/chainlink/v2/core/services/webhook" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows" "github.com/smartcontractkit/chainlink/v2/core/sessions" "github.com/smartcontractkit/chainlink/v2/core/static" "github.com/smartcontractkit/chainlink/v2/core/store/migrate" @@ -111,6 +112,10 @@ func initGlobals(cfgProm config.Prometheus, cfgTracing config.Tracing, cfgTeleme AuthPublicKeyHex: csaPubKeyHex, AuthHeaders: beholderAuthHeaders, } + // note: due to the OTEL specification, all histogram buckets + // must be defined when the beholder client is created + clientCfg.MetricViews = append(clientCfg.MetricViews, workflows.MetricViews()...) + if tracingCfg.Enabled { clientCfg.TraceSpanExporter, err = tracingCfg.NewSpanExporter() if err != nil { diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 65d63f2a1df..d468907c550 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -26,7 +26,7 @@ require ( github.com/prometheus/client_golang v1.20.5 github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chainlink-automation v0.8.1 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f github.com/smartcontractkit/chainlink/deployment v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241106193309-5560cd76211a github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 1f1374bc11d..225ae3b9191 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1142,8 +1142,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d h1:5XKarlliHXVVAhpCeOAx/TRU7eWsJ3tkqRI3I6Cc0SU= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f h1:hH+cAG2zt+WK4I2m572LXAnAJg3wtGEAwzBKR8FiXo8= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/core/services/workflows/monitoring.go b/core/services/workflows/monitoring.go index 205ce529c28..8457dadeb60 100644 --- a/core/services/workflows/monitoring.go +++ b/core/services/workflows/monitoring.go @@ -5,6 +5,7 @@ import ( "fmt" "go.opentelemetry.io/otel/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/metrics" @@ -135,6 +136,37 @@ func initMonitoringResources() (em *engineMetrics, err error) { return em, nil } +// Note: due to the OTEL specification, all histogram buckets +// Must be defined when the beholder client is created +func MetricViews() []sdkmetric.View { + return []sdkmetric.View{ + sdkmetric.NewView( + sdkmetric.Instrument{Name: "platform_engine_workflow_earlyexit_time_seconds"}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{ + Boundaries: []float64{0, 1, 10, 100}, + }}, + ), + sdkmetric.NewView( + sdkmetric.Instrument{Name: "platform_engine_workflow_completed_time_seconds"}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{ + Boundaries: []float64{0, 100, 1000, 10_000, 50_000, 100_0000, 500_000}, + }}, + ), + sdkmetric.NewView( + sdkmetric.Instrument{Name: "platform_engine_workflow_error_time_seconds"}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{ + Boundaries: []float64{0, 20, 60, 120, 240}, + }}, + ), + sdkmetric.NewView( + sdkmetric.Instrument{Name: "platform_engine_workflow_step_time_seconds"}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{ + Boundaries: []float64{0, 20, 60, 120, 240}, + }}, + ), + } +} + // workflowsMetricLabeler wraps monitoring.MetricsLabeler to provide workflow specific utilities // for monitoring resources type workflowsMetricLabeler struct { diff --git a/deployment/go.mod b/deployment/go.mod index b914f76d59b..7908b6a8048 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -25,7 +25,7 @@ require ( github.com/smartcontractkit/ccip-owner-contracts v0.0.0-20240926212305-a6deabdfce86 github.com/smartcontractkit/chain-selectors v1.0.31 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.13 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 diff --git a/deployment/go.sum b/deployment/go.sum index d740931e9a4..fec529e51c6 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1411,8 +1411,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d h1:5XKarlliHXVVAhpCeOAx/TRU7eWsJ3tkqRI3I6Cc0SU= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f h1:hH+cAG2zt+WK4I2m572LXAnAJg3wtGEAwzBKR8FiXo8= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/go.mod b/go.mod index f1d3b1b71e5..ad71fa75be3 100644 --- a/go.mod +++ b/go.mod @@ -79,7 +79,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.31 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db github.com/smartcontractkit/chainlink-feeds v0.1.1 @@ -106,6 +106,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0 go.opentelemetry.io/otel v1.31.0 go.opentelemetry.io/otel/metric v1.31.0 + go.opentelemetry.io/otel/sdk/metric v1.31.0 go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 @@ -364,7 +365,6 @@ require ( go.opentelemetry.io/otel/log v0.6.0 // indirect go.opentelemetry.io/otel/sdk v1.31.0 // indirect go.opentelemetry.io/otel/sdk/log v0.6.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/ratelimit v0.3.1 // indirect golang.org/x/arch v0.11.0 // indirect diff --git a/go.sum b/go.sum index cdf7fb3805d..42de8ab4a72 100644 --- a/go.sum +++ b/go.sum @@ -1125,8 +1125,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d h1:5XKarlliHXVVAhpCeOAx/TRU7eWsJ3tkqRI3I6Cc0SU= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f h1:hH+cAG2zt+WK4I2m572LXAnAJg3wtGEAwzBKR8FiXo8= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index b7ef4de1bae..b9c6f213277 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -41,7 +41,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.31 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.18 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 9cc6814f3cc..6914a06501f 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1432,8 +1432,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d h1:5XKarlliHXVVAhpCeOAx/TRU7eWsJ3tkqRI3I6Cc0SU= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f h1:hH+cAG2zt+WK4I2m572LXAnAJg3wtGEAwzBKR8FiXo8= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 847fc58b543..34717eea023 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -19,7 +19,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.33.0 github.com/slack-go/slack v0.15.0 - github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d + github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.18 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.9 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index c4215f0c19e..bcb3680e9c5 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1423,8 +1423,8 @@ github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgB github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc= github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d h1:5XKarlliHXVVAhpCeOAx/TRU7eWsJ3tkqRI3I6Cc0SU= -github.com/smartcontractkit/chainlink-common v0.3.1-0.20241204184525-29871ced7b4d/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f h1:hH+cAG2zt+WK4I2m572LXAnAJg3wtGEAwzBKR8FiXo8= +github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206011233-b6684ee6508f/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0= github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk= github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241202141438-a90db35252db h1:N1RH1hSr2ACzOFc9hkCcjE8pRBTdcU3p8nsTJByaLes=