From 01ce8d7d30fb9ae9e9cc2418aadffbc521f2e2ba Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Thu, 27 Jul 2023 10:15:33 -0400 Subject: [PATCH 1/2] Add opentelemetry to shared web Use Jaeger for tracing --- dev/docker-compose.yml | 12 +++++++++++ src/SharedWeb/Health/DiagnosticsConfig.cs | 21 +++++++++++++++++++ src/SharedWeb/SharedWeb.csproj | 6 ++++++ .../Utilities/ServiceCollectionExtensions.cs | 13 ++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 src/SharedWeb/Health/DiagnosticsConfig.cs diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 4d75f2007186..f18a11f838db 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -105,6 +105,18 @@ services: - "${IDENTITY_PROXY_PORT}:${IDENTITY_PROXY_PORT}" profiles: - proxy + + jaeger: + image: jaegertracing/all-in-one:latest + container_name: jaeger + environment: + COLLECTOR_OTLP_ENABLED: true + ports: + - "16686:16686" + - "4317:4317" + - "4318:4318" + profiles: + - jaeger volumes: edgesql_dev_data: diff --git a/src/SharedWeb/Health/DiagnosticsConfig.cs b/src/SharedWeb/Health/DiagnosticsConfig.cs new file mode 100644 index 000000000000..8808c19dbf1a --- /dev/null +++ b/src/SharedWeb/Health/DiagnosticsConfig.cs @@ -0,0 +1,21 @@ +using System.Diagnostics; +using Bit.Core.Settings; + +namespace Bit.SharedWeb.Health; + +public class DiagnosticsConfig +{ + public string ServiceName { get; } + public ActivitySource ActivitySource { get; } + + public static DiagnosticsConfig For(GlobalSettings globalSettings) + { + return new DiagnosticsConfig(globalSettings); + } + + private DiagnosticsConfig(GlobalSettings globalSettings) + { + ServiceName = globalSettings.ProjectName ?? "Bitwarden"; + ActivitySource = new ActivitySource(ServiceName); + } +} diff --git a/src/SharedWeb/SharedWeb.csproj b/src/SharedWeb/SharedWeb.csproj index e25cc988f10c..d46d1ff76879 100644 --- a/src/SharedWeb/SharedWeb.csproj +++ b/src/SharedWeb/SharedWeb.csproj @@ -10,4 +10,10 @@ + + + + + + diff --git a/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs b/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs index f38130574578..c3de677ceb9c 100644 --- a/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs +++ b/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs @@ -35,6 +35,7 @@ using Bit.Core.Vault.Services; using Bit.Infrastructure.Dapper; using Bit.Infrastructure.EntityFramework; +using Bit.SharedWeb.Health; using DnsClient; using IdentityModel; using LaunchDarkly.Sdk.Server; @@ -57,6 +58,8 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; using Serilog.Context; using StackExchange.Redis; using NoopRepos = Bit.Core.Repositories.Noop; @@ -235,6 +238,16 @@ public static void AddDefaultServices(this IServiceCollection services, GlobalSe services.AddWebAuthn(globalSettings); // Required for HTTP calls services.AddHttpClient(); + // Required for open telemetry + var diagnosticsConfig = DiagnosticsConfig.For(globalSettings); + services.AddOpenTelemetry().WithTracing(tracerProviderBuilder => + tracerProviderBuilder + .AddSource(diagnosticsConfig.ActivitySource.Name) + .ConfigureResource(resource => resource + .AddService(diagnosticsConfig.ServiceName)) + .AddAspNetCoreInstrumentation() + .AddOtlpExporter()); + services.AddSingleton(); services.AddSingleton((serviceProvider) => From d1a83cfe0ec6d9d1d5a394be2d9f5c2d26d70a60 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Thu, 27 Jul 2023 17:28:36 -0400 Subject: [PATCH 2/2] Add open telemetry Add diagnostics for aspdotnet core, middleware, sqlserver, ef core, http clients, and action/result filters for controller actions --- src/Api/Startup.cs | 7 +- src/Core/Core.csproj | 6 ++ src/Core/Utilities/TracingAttribute.cs | 71 +++++++++++++++++++ .../Properties/launchSettings.json | 8 +++ .../Infrastructure.EntityFramework.csproj | 1 + .../MiddlewareAnalysisDiagnosticAdapter.cs | 57 +++++++++++++++ src/SharedWeb/SharedWeb.csproj | 1 + src/SharedWeb/Utilities/DiagnosticsHelpers.cs | 18 +++++ .../Utilities/ServiceCollectionExtensions.cs | 23 +++--- 9 files changed, 183 insertions(+), 9 deletions(-) create mode 100644 src/Core/Utilities/TracingAttribute.cs create mode 100644 src/SharedWeb/Health/MiddlewareAnalysisDiagnosticAdapter.cs create mode 100644 src/SharedWeb/Utilities/DiagnosticsHelpers.cs diff --git a/src/Api/Startup.cs b/src/Api/Startup.cs index fd2a4dbe6f77..6a42be7b266e 100644 --- a/src/Api/Startup.cs +++ b/src/Api/Startup.cs @@ -199,6 +199,7 @@ public void ConfigureServices(IServiceCollection services) { config.Conventions.Add(new ApiExplorerGroupConvention()); config.Conventions.Add(new PublicApiControllersModelConvention()); + config.Filters.Add(new TracingAttribute()); // Add tracing to controller actions and results }); services.AddSwagger(globalSettings); @@ -217,11 +218,15 @@ public void Configure( IWebHostEnvironment env, IHostApplicationLifetime appLifetime, GlobalSettings globalSettings, - ILogger logger) + ILogger logger, + IServiceProvider serviceProvider) { IdentityModelEventSource.ShowPII = true; app.UseSerilog(env, appLifetime, globalSettings); + // Middleware telemetry + DiagnosticsHelpers.AddMiddlewareDiagnostics(serviceProvider); + // Add general security headers app.UseMiddleware(); diff --git a/src/Core/Core.csproj b/src/Core/Core.csproj index 1eca173c9946..2cead76f9d23 100644 --- a/src/Core/Core.csproj +++ b/src/Core/Core.csproj @@ -35,6 +35,7 @@ + @@ -42,6 +43,11 @@ + + + + + diff --git a/src/Core/Utilities/TracingAttribute.cs b/src/Core/Utilities/TracingAttribute.cs new file mode 100644 index 000000000000..61892e307efb --- /dev/null +++ b/src/Core/Utilities/TracingAttribute.cs @@ -0,0 +1,71 @@ +#nullable enable + +using System.Diagnostics; +using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.Extensions.DependencyInjection; + +namespace Bit.Core.Utilities; + +public class TracingAttribute : ActionFilterAttribute +{ + private Activity? _actionActivity; + private Activity? _resultActivity; + + public override void OnActionExecuting(ActionExecutingContext context) + { + var activitySource = context.HttpContext.RequestServices.GetService(); + + if (activitySource == null) + { + return; + } + + var activityName = $"{context.ActionDescriptor.DisplayName}_action"; + _actionActivity = activitySource.StartActivity(activityName, ActivityKind.Server); + } + + public override void OnActionExecuted(ActionExecutedContext context) + { + if (_actionActivity == null) + { + return; + } + + if (context.Exception != null) + { + _actionActivity.AddTag("exception", context.Exception.ToString()); + _actionActivity.AddTag("exceptionStackTrace", context.Exception.StackTrace); + } + + _actionActivity.Stop(); + } + + public override void OnResultExecuting(ResultExecutingContext context) + { + var activitySource = context.HttpContext.RequestServices.GetService(); + + if (activitySource == null) + { + return; + } + + var activityName = $"{context.ActionDescriptor.DisplayName}_result"; + _resultActivity = activitySource.StartActivity(activityName, ActivityKind.Server); + } + + public override void OnResultExecuted(ResultExecutedContext context) + { + if (_resultActivity == null) + { + return; + } + + if (context.Exception != null) + { + _resultActivity.AddTag("exception", context.Exception.ToString()); + _resultActivity.AddTag("exceptionStackTrace", context.Exception.StackTrace); + } + + _resultActivity.Stop(); + } +} diff --git a/src/EventsProcessor/Properties/launchSettings.json b/src/EventsProcessor/Properties/launchSettings.json index f772b225dfd8..b3dbfd2aee59 100644 --- a/src/EventsProcessor/Properties/launchSettings.json +++ b/src/EventsProcessor/Properties/launchSettings.json @@ -20,6 +20,14 @@ "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } + }, + "EventsProcessor-SelfHost": { + "commandName": "Project", + "applicationUrl": "http://localhost:54104/", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "developSelfHosted": "true" + } } } } diff --git a/src/Infrastructure.EntityFramework/Infrastructure.EntityFramework.csproj b/src/Infrastructure.EntityFramework/Infrastructure.EntityFramework.csproj index 7b008ba85c6d..449f6860cc6d 100644 --- a/src/Infrastructure.EntityFramework/Infrastructure.EntityFramework.csproj +++ b/src/Infrastructure.EntityFramework/Infrastructure.EntityFramework.csproj @@ -7,6 +7,7 @@ + diff --git a/src/SharedWeb/Health/MiddlewareAnalysisDiagnosticAdapter.cs b/src/SharedWeb/Health/MiddlewareAnalysisDiagnosticAdapter.cs new file mode 100644 index 000000000000..f4d8d9fd61be --- /dev/null +++ b/src/SharedWeb/Health/MiddlewareAnalysisDiagnosticAdapter.cs @@ -0,0 +1,57 @@ +#nullable enable + +using System.Collections.Concurrent; +using System.Diagnostics; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DiagnosticAdapter; + +namespace Bit.SharedWeb.Health; + +public class MiddlewareAnalysisDiagnosticAdapter +{ + private readonly ActivitySource _activitySource; + private readonly ConcurrentDictionary _activities = new(); + + public MiddlewareAnalysisDiagnosticAdapter(ActivitySource activitySource) + { + _activitySource = activitySource; + } + + [DiagnosticName("Microsoft.AspNetCore.MiddlewareAnalysis.MiddlewareStarting")] + public void OnMiddlewareStarting(HttpContext httpContext, string name, Guid instance, long timestamp) + { + if (name == "Microsoft.AspNetCore.MiddlewareAnalysis.AnalysisMiddleware") + { + return; + } + + if (_activitySource.StartActivity(name, ActivityKind.Server) is Activity activity) + { + _activities[name] = activity; + } + } + + [DiagnosticName("Microsoft.AspNetCore.MiddlewareAnalysis.MiddlewareException")] + public void OnMiddlewareException(Exception exception, HttpContext httpContext, string name, Guid instance, long timestamp, long duration) + { + if (name == "Microsoft.AspNetCore.MiddlewareAnalysis.AnalysisMiddleware" || !_activities.ContainsKey(name)) + { + return; + } + _activities[name].AddTag("exception", exception.ToString()); + _activities[name].AddTag("exceptionStackTrace", exception.StackTrace); + + _activities[name].Stop(); + } + + [DiagnosticName("Microsoft.AspNetCore.MiddlewareAnalysis.MiddlewareFinished")] + public void OnMiddlewareFinished(HttpContext httpContext, string name, Guid instance, long timestamp, long duration) + { + if (name == "Microsoft.AspNetCore.MiddlewareAnalysis.AnalysisMiddleware" || !_activities.ContainsKey(name)) + { + return; + } + _activities[name].Stop(); + } + +} diff --git a/src/SharedWeb/SharedWeb.csproj b/src/SharedWeb/SharedWeb.csproj index d46d1ff76879..f6fafb9cb713 100644 --- a/src/SharedWeb/SharedWeb.csproj +++ b/src/SharedWeb/SharedWeb.csproj @@ -11,6 +11,7 @@ + diff --git a/src/SharedWeb/Utilities/DiagnosticsHelpers.cs b/src/SharedWeb/Utilities/DiagnosticsHelpers.cs new file mode 100644 index 000000000000..6f37dab4d66b --- /dev/null +++ b/src/SharedWeb/Utilities/DiagnosticsHelpers.cs @@ -0,0 +1,18 @@ +#nullable enable + +using System.Diagnostics; +using Bit.SharedWeb.Health; +using Microsoft.Extensions.DependencyInjection; + +namespace Bit.SharedWeb.Utilities; + +public static class DiagnosticsHelpers +{ + public static void AddMiddlewareDiagnostics(IServiceProvider serviceProvider) + { + var listener = serviceProvider.GetRequiredService(); + var activitySource = serviceProvider.GetRequiredService(); + + listener.SubscribeWithAdapter(new MiddlewareAnalysisDiagnosticAdapter(activitySource)); + } +} diff --git a/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs b/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs index c3de677ceb9c..7211a79c5ffa 100644 --- a/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs +++ b/src/SharedWeb/Utilities/ServiceCollectionExtensions.cs @@ -49,6 +49,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.AspNetCore.Identity; +using Microsoft.AspNetCore.MiddlewareAnalysis; using Microsoft.AspNetCore.Mvc.Localization; using Microsoft.Azure.Cosmos.Fluent; using Microsoft.Extensions.Caching.Cosmos; @@ -238,16 +239,22 @@ public static void AddDefaultServices(this IServiceCollection services, GlobalSe services.AddWebAuthn(globalSettings); // Required for HTTP calls services.AddHttpClient(); - // Required for open telemetry + + // Add open telemetry var diagnosticsConfig = DiagnosticsConfig.For(globalSettings); + services.AddSingleton(diagnosticsConfig.ActivitySource); + services.AddMiddlewareAnalysis(); + services.Insert(0, ServiceDescriptor.Transient()); // Insert at beginning to catch all middleware services.AddOpenTelemetry().WithTracing(tracerProviderBuilder => - tracerProviderBuilder - .AddSource(diagnosticsConfig.ActivitySource.Name) - .ConfigureResource(resource => resource - .AddService(diagnosticsConfig.ServiceName)) - .AddAspNetCoreInstrumentation() - .AddOtlpExporter()); - + tracerProviderBuilder + .AddSource(diagnosticsConfig.ActivitySource.Name) + .ConfigureResource(resource => resource + .AddService(diagnosticsConfig.ServiceName)) + .AddAspNetCoreInstrumentation() + .AddSqlClientInstrumentation() + .AddEntityFrameworkCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddOtlpExporter()); services.AddSingleton(); services.AddSingleton((serviceProvider) =>