Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(infra): add opentelemetry metric instrumentation (#1375)
* chore: merge main branch * feat: add codes for automatic instrumentation in client api * feat(infra): rebase main branch * chore(infra): merge tracer into metrics * Revert "chore(infra): merge tracer into metrics" This reverts commit 5c9d899. * chore(infra): merge mian * Revert "feat(infra): rebase main branch" This reverts commit a65f505. * feat(infra): iris memory metrics * refactor(iris): separate application logic from metric-related code * feat(iris): add CPU metrics in iris * chore(iris): add comment of asynchronous metrics * feat(infra): add backend opentelemtry instrumentation for client and admin api * feat(infra): install packages for opentelemetry instrumentation * feat(infra): change iris metric exporter url from env not specifying url * chore(infra): add env of stage-otel * fix(infra): delete grafana agent otlp endpoint variable * feat(infra): install nest package for opentelemetry metric instrumentation * feat(infra): install nest package for opentelemetry metric instrumentation * feat(infra): install go package for iris opentelemetry metric instrumentation * feat(infra): add metric.go functions to main which trace.go functions are already included * feat(infra): add metric.ts functions to main which trace is included for admin and client api * chore: resolve otel dependency * chore: resolve otel dependency for iris * fix(infra): delete files which not in main branch --------- Co-authored-by: k1g99 <[email protected]> Co-authored-by: Goathoon <[email protected]>
- Loading branch information
3 people
authored
Mar 14, 2024
1 parent
f3d0f3d
commit 4a586dd
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http' | ||
| import { HostMetrics } from '@opentelemetry/host-metrics' | ||
| import { Resource } from '@opentelemetry/resources' | ||
| import { | ||
| MeterProvider, | ||
| PeriodicExportingMetricReader | ||
| } from '@opentelemetry/sdk-metrics' | ||
| import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions' | ||
|
|
||
| const startMetricsExporter = () => { | ||
| const options = { | ||
| url: 'http://' + process.env.OTEL_EXPORTER_OTLP_ENDPOINT + '/v1/metrics', // Grafana Agent Metric을 받는 url | ||
| headers: {}, | ||
| concurrencyLimit: 5 | ||
| } | ||
| const exporter = new OTLPMetricExporter(options) | ||
|
|
||
| const resource = new Resource({ | ||
| [SemanticResourceAttributes.SERVICE_NAME]: 'backend-admin-metric', | ||
| [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0', | ||
| environment: 'production' // 사용자 정의 속성 | ||
| }) | ||
|
|
||
| // Creates MeterProvider and installs the exporter as a MetricReader | ||
| const meterProvider = new MeterProvider({ | ||
| resource: resource | ||
| }) | ||
| meterProvider.addMetricReader( | ||
| new PeriodicExportingMetricReader({ | ||
| exporter, | ||
| exportIntervalMillis: 1000 | ||
| }) | ||
| ) | ||
|
|
||
| const hostMetrics = new HostMetrics({ | ||
| meterProvider, | ||
| name: 'backend-admin-host-metric' | ||
| }) | ||
| hostMetrics.start() | ||
| } | ||
|
|
||
| export default startMetricsExporter | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http' | ||
| import { HostMetrics } from '@opentelemetry/host-metrics' | ||
| import { Resource } from '@opentelemetry/resources' | ||
| import { | ||
| MeterProvider, | ||
| PeriodicExportingMetricReader | ||
| } from '@opentelemetry/sdk-metrics' | ||
| import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions' | ||
|
|
||
| const startMetricsExporter = () => { | ||
| const options = { | ||
| url: 'http://' + process.env.OTEL_EXPORTER_OTLP_ENDPOINT + '/v1/metrics', // Grafana Agent Metric을 받는 url | ||
| headers: {}, | ||
| concurrencyLimit: 5 | ||
| } | ||
| const exporter = new OTLPMetricExporter(options) | ||
|
|
||
| const resource = new Resource({ | ||
| [SemanticResourceAttributes.SERVICE_NAME]: 'backend-client-metric', | ||
| [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0', | ||
| environment: 'production' // 사용자 정의 속성 | ||
| }) | ||
|
|
||
| // Creates MeterProvider and installs the exporter as a MetricReader | ||
| const meterProvider = new MeterProvider({ | ||
| resource: resource | ||
| }) | ||
| meterProvider.addMetricReader( | ||
| new PeriodicExportingMetricReader({ | ||
| exporter, | ||
| exportIntervalMillis: 1000 | ||
| }) | ||
| ) | ||
|
|
||
| const hostMetrics = new HostMetrics({ | ||
| meterProvider, | ||
| name: 'backend-client-host-metric' | ||
| }) | ||
| hostMetrics.start() | ||
| } | ||
|
|
||
| export default startMetricsExporter | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| package observability | ||
|
|
||
| import ( | ||
| "context" | ||
| "os" | ||
| "runtime" | ||
| "time" | ||
|
|
||
| "github.com/shirou/gopsutil/cpu" | ||
| "go.opentelemetry.io/otel" | ||
| "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" | ||
| "go.opentelemetry.io/otel/metric" | ||
| sdkmetric "go.opentelemetry.io/otel/sdk/metric" | ||
| "go.opentelemetry.io/otel/sdk/resource" | ||
| semconv "go.opentelemetry.io/otel/semconv/v1.24.0" | ||
| ) | ||
|
|
||
| const defaultMetricDuration time.Duration = 3 * time.Second | ||
|
|
||
| func SetGlobalMeterProvider() { | ||
| // Create resource. | ||
| res, err := newMetricResource() | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| // Create a meter provider. | ||
| // You can pass this instance directly to your instrumented code if it | ||
| // accepts a MeterProvider instance. | ||
| meterProvider, err := newMeterProvider(res, defaultMetricDuration) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| // Register as global meter provider so that it can be used via otel.Meter | ||
| // and accessed using otel.GetMeterProvider. | ||
| // Most instrumentation libraries use the global meter provider as default. | ||
| // If the global meter provider is not set then a no-op implementation | ||
| // is used, which fails to generate data. | ||
| otel.SetMeterProvider(meterProvider) | ||
| } | ||
|
|
||
| func newMetricResource() (*resource.Resource, error) { | ||
| return resource.Merge(resource.Default(), | ||
| resource.NewWithAttributes(semconv.SchemaURL, | ||
| semconv.ServiceName("iris-metric"), | ||
| semconv.ServiceVersion("0.1.0"), | ||
| )) | ||
| } | ||
|
|
||
| func newMeterProvider(res *resource.Resource, second time.Duration) (*sdkmetric.MeterProvider, error) { | ||
| // Use OLTP Exporter for Grafana Agent (Recommended) | ||
| entryPoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") | ||
| otlpExporter, err := otlpmetrichttp.New(context.Background(), otlpmetrichttp.WithEndpointURL("http://"+entryPoint+"/v1/metrics")) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| meterProvider := sdkmetric.NewMeterProvider( | ||
| sdkmetric.WithResource(res), | ||
| sdkmetric.WithReader(sdkmetric.NewPeriodicReader(otlpExporter, | ||
| // Define duration of Metric | ||
| sdkmetric.WithInterval(second))), | ||
| ) | ||
| return meterProvider, nil | ||
| } | ||
|
|
||
| // TODO - Int64ObservableGauge를 Meter로 추상화할 수 있었으면 합니다. | ||
| func GetMemoryMeter(meter metric.Meter) { | ||
| if _, err := meter.Int64ObservableGauge( | ||
| "memory.heap", | ||
| metric.WithDescription( | ||
| "Memory usage of the allocated heap objects.", | ||
| ), | ||
| metric.WithUnit("By"), // UCUM 규약의 Byte | ||
| metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error { | ||
| var m runtime.MemStats | ||
| runtime.ReadMemStats(&m) | ||
| o.Observe(int64(m.HeapAlloc)) | ||
| return nil | ||
| }), | ||
| ); err != nil { | ||
| panic(err) | ||
| } | ||
| } | ||
|
|
||
| func GetCPUMeter(meter metric.Meter, duration time.Duration) { | ||
| // 일단 전체 CPU Usage를 추적합니다. | ||
| // 추후에, CPU별로 Usage 추적이 필요할 경우 true로 설정합니다. | ||
|
|
||
| //Cpu시간을 cpu.comman함수로 ctx를 중지시키고, Duration 차이에 해당하는 CPU time의 차를 계산합니다. | ||
| // busy := t.User + t.System + t.Nice + t.Iowait + t.Irq + t.Softirq + t.Steal 시간의 전체를 CPU usage 계산에 사용합니다. | ||
| if _, err := meter.Float64ObservableGauge( | ||
| "cpu.usage", | ||
| metric.WithDescription( | ||
| "All CPU", | ||
| ), | ||
| metric.WithUnit("%"), | ||
| metric.WithFloat64Callback(func(_ context.Context, o metric.Float64Observer) error { | ||
| cpuPercent, _ := cpu.Percent(duration, false) | ||
| o.Observe(float64(cpuPercent[0])) | ||
| return nil | ||
| }), | ||
| ); err != nil { | ||
| panic(err) | ||
| } | ||
| } |
Oops, something went wrong.