Add opentelemetry

dev/docker-compose.yml

@@ -105,6 +105,18 @@ services:
       - "${IDENTITY_PROXY_PORT}:${IDENTITY_PROXY_PORT}"
     profiles:
       - proxy
+
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    container_name: jaeger
+    environment:
+      COLLECTOR_OTLP_ENABLED: true
+    ports:
+      - "16686:16686"
+      - "4317:4317"
+      - "4318:4318"
+    profiles:
+      - jaeger
 
 volumes:
   edgesql_dev_data:

src/Api/Startup.cs

@@ -199,6 +199,7 @@ public void ConfigureServices(IServiceCollection services)
         {
             config.Conventions.Add(new ApiExplorerGroupConvention());
             config.Conventions.Add(new PublicApiControllersModelConvention());
+            config.Filters.Add(new TracingAttribute()); // Add tracing to controller actions and results
         });
 
         services.AddSwagger(globalSettings);
@@ -217,11 +218,15 @@ public void Configure(
         IWebHostEnvironment env,
         IHostApplicationLifetime appLifetime,
         GlobalSettings globalSettings,
-        ILogger<Startup> logger)
+        ILogger<Startup> logger,
+        IServiceProvider serviceProvider)
     {
         IdentityModelEventSource.ShowPII = true;
         app.UseSerilog(env, appLifetime, globalSettings);
 
+        // Middleware telemetry
+        DiagnosticsHelpers.AddMiddlewareDiagnostics(serviceProvider);
+
         // Add general security headers
         app.UseMiddleware<SecurityHeadersMiddleware>();
 

src/Core/Core.csproj

@@ -35,13 +35,19 @@
     <PackageReference Include="Handlebars.Net" Version="2.1.6" />
     <PackageReference Include="MailKit" Version="4.6.0" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.31" />
+    <PackageReference Include="Microsoft.AspNetCore.MiddlewareAnalysis" Version="6.0.20" />
     <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.39.1" />
     <PackageReference Include="Microsoft.Azure.NotificationHubs" Version="4.2.0" />
     <PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.1" />
     <PackageReference Include="Microsoft.Extensions.Caching.Cosmos" Version="1.6.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Identity.Stores" Version="6.0.31" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.5.1" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.5.1" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.5.1-beta.1" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.5.1-beta.1" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.SqlClient" Version="1.5.1-beta.1" />
     <PackageReference Include="Quartz" Version="3.9.0" />
     <PackageReference Include="SendGrid" Version="9.29.3" />
     <PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />

src/Core/Utilities/TracingAttribute.cs

@@ -0,0 +1,71 @@
+#nullable enable
+
+using System.Diagnostics;
+using Microsoft.AspNetCore.Mvc.Filters;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Bit.Core.Utilities;
+
+public class TracingAttribute : ActionFilterAttribute
+{
+    private Activity? _actionActivity;
+    private Activity? _resultActivity;
+
+    public override void OnActionExecuting(ActionExecutingContext context)
+    {
+        var activitySource = context.HttpContext.RequestServices.GetService<ActivitySource>();
+
+        if (activitySource == null)
+        {
+            return;
+        }
+
+        var activityName = $"{context.ActionDescriptor.DisplayName}_action";
+        _actionActivity = activitySource.StartActivity(activityName, ActivityKind.Server);
+    }
+
+    public override void OnActionExecuted(ActionExecutedContext context)
+    {
+        if (_actionActivity == null)
+        {
+            return;
+        }
+
+        if (context.Exception != null)
+        {
+            _actionActivity.AddTag("exception", context.Exception.ToString());
+            _actionActivity.AddTag("exceptionStackTrace", context.Exception.StackTrace);
+        }
+
+        _actionActivity.Stop();
+    }
+
+    public override void OnResultExecuting(ResultExecutingContext context)
+    {
+        var activitySource = context.HttpContext.RequestServices.GetService<ActivitySource>();
+
+        if (activitySource == null)
+        {
+            return;
+        }
+
+        var activityName = $"{context.ActionDescriptor.DisplayName}_result";
+        _resultActivity = activitySource.StartActivity(activityName, ActivityKind.Server);
+    }
+
+    public override void OnResultExecuted(ResultExecutedContext context)
+    {
+        if (_resultActivity == null)
+        {
+            return;
+        }
+
+        if (context.Exception != null)
+        {
+            _resultActivity.AddTag("exception", context.Exception.ToString());
+            _resultActivity.AddTag("exceptionStackTrace", context.Exception.StackTrace);
+        }
+
+        _resultActivity.Stop();
+    }
+}

src/EventsProcessor/Properties/launchSettings.json

@@ -20,6 +20,14 @@
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
       }
+    },
+    "EventsProcessor-SelfHost": {
+      "commandName": "Project",
+      "applicationUrl": "http://localhost:54104/",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "developSelfHosted": "true"
+      }
     }
   }
 }

src/Infrastructure.EntityFramework/Infrastructure.EntityFramework.csproj

@@ -7,6 +7,7 @@
       <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="7.0.20" />
       <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="7.0.20" />
       <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="7.0.18" />
+      <PackageReference Include="OpenTelemetry.Instrumentation.EntityFrameworkCore" Version="1.0.0-beta.7" />
       <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="7.0.0" />
       <PackageReference Include="linq2db.EntityFrameworkCore" Version="7.7.0" />
     </ItemGroup>

src/SharedWeb/Health/DiagnosticsConfig.cs

@@ -0,0 +1,21 @@
+using System.Diagnostics;
+using Bit.Core.Settings;
+
+namespace Bit.SharedWeb.Health;
+
+public class DiagnosticsConfig
+{
+    public string ServiceName { get; }
+    public ActivitySource ActivitySource { get; }
+
+    public static DiagnosticsConfig For(GlobalSettings globalSettings)
+    {
+        return new DiagnosticsConfig(globalSettings);
+    }
+
+    private DiagnosticsConfig(GlobalSettings globalSettings)
+    {
+        ServiceName = globalSettings.ProjectName ?? "Bitwarden";
+        ActivitySource = new ActivitySource(ServiceName);
+    }
+}

src/SharedWeb/Health/MiddlewareAnalysisDiagnosticAdapter.cs

@@ -0,0 +1,57 @@
+#nullable enable
+
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.DiagnosticAdapter;
+
+namespace Bit.SharedWeb.Health;
+
+public class MiddlewareAnalysisDiagnosticAdapter
+{
+    private readonly ActivitySource _activitySource;
+    private readonly ConcurrentDictionary<string, Activity> _activities = new();
+
+    public MiddlewareAnalysisDiagnosticAdapter(ActivitySource activitySource)
+    {
+        _activitySource = activitySource;
+    }
+
+    [DiagnosticName("Microsoft.AspNetCore.MiddlewareAnalysis.MiddlewareStarting")]
+    public void OnMiddlewareStarting(HttpContext httpContext, string name, Guid instance, long timestamp)
+    {
+        if (name == "Microsoft.AspNetCore.MiddlewareAnalysis.AnalysisMiddleware")
+        {
+            return;
+        }
+
+        if (_activitySource.StartActivity(name, ActivityKind.Server) is Activity activity)
+        {
+            _activities[name] = activity;
+        }
+    }
+
+    [DiagnosticName("Microsoft.AspNetCore.MiddlewareAnalysis.MiddlewareException")]
+    public void OnMiddlewareException(Exception exception, HttpContext httpContext, string name, Guid instance, long timestamp, long duration)
+    {
+        if (name == "Microsoft.AspNetCore.MiddlewareAnalysis.AnalysisMiddleware" || !_activities.ContainsKey(name))
+        {
+            return;
+        }
+        _activities[name].AddTag("exception", exception.ToString());
+        _activities[name].AddTag("exceptionStackTrace", exception.StackTrace);
+
+        _activities[name].Stop();
+    }
+
+    [DiagnosticName("Microsoft.AspNetCore.MiddlewareAnalysis.MiddlewareFinished")]
+    public void OnMiddlewareFinished(HttpContext httpContext, string name, Guid instance, long timestamp, long duration)
+    {
+        if (name == "Microsoft.AspNetCore.MiddlewareAnalysis.AnalysisMiddleware" || !_activities.ContainsKey(name))
+        {
+            return;
+        }
+        _activities[name].Stop();
+    }
+
+}

src/SharedWeb/SharedWeb.csproj

@@ -10,4 +10,11 @@
     <PackageReference Include="Swashbuckle.AspNetCore.SwaggerGen" Version="6.5.0" />
   </ItemGroup>
 
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.DiagnosticAdapter" Version="3.1.32" />
+      <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.5.1" />
+      <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.5.1" />
+      <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.5.1-beta.1" />
+    </ItemGroup>
+
 </Project>

src/SharedWeb/Utilities/DiagnosticsHelpers.cs

@@ -0,0 +1,18 @@
+#nullable enable
+
+using System.Diagnostics;
+using Bit.SharedWeb.Health;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Bit.SharedWeb.Utilities;
+
+public static class DiagnosticsHelpers
+{
+    public static void AddMiddlewareDiagnostics(IServiceProvider serviceProvider)
+    {
+        var listener = serviceProvider.GetRequiredService<DiagnosticListener>();
+        var activitySource = serviceProvider.GetRequiredService<ActivitySource>();
+
+        listener.SubscribeWithAdapter(new MiddlewareAnalysisDiagnosticAdapter(activitySource));
+    }
+}

src/SharedWeb/Utilities/ServiceCollectionExtensions.cs

@@ -35,6 +35,7 @@
 using Bit.Core.Vault.Services;
 using Bit.Infrastructure.Dapper;
 using Bit.Infrastructure.EntityFramework;
+using Bit.SharedWeb.Health;
 using DnsClient;
 using IdentityModel;
 using LaunchDarkly.Sdk.Server;
@@ -48,6 +49,7 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.AspNetCore.Identity;
+using Microsoft.AspNetCore.MiddlewareAnalysis;
 using Microsoft.AspNetCore.Mvc.Localization;
 using Microsoft.Azure.Cosmos.Fluent;
 using Microsoft.Extensions.Caching.Cosmos;
@@ -57,6 +59,8 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
 using Serilog.Context;
 using StackExchange.Redis;
 using NoopRepos = Bit.Core.Repositories.Noop;
@@ -236,6 +240,22 @@ public static void AddDefaultServices(this IServiceCollection services, GlobalSe
         // Required for HTTP calls
         services.AddHttpClient();
 
+        // Add open telemetry
+        var diagnosticsConfig = DiagnosticsConfig.For(globalSettings);
+        services.AddSingleton(diagnosticsConfig.ActivitySource);
+        services.AddMiddlewareAnalysis();
+        services.Insert(0, ServiceDescriptor.Transient<IStartupFilter, AnalysisStartupFilter>()); // Insert at beginning to catch all middleware
+        services.AddOpenTelemetry().WithTracing(tracerProviderBuilder =>
+                tracerProviderBuilder
+                    .AddSource(diagnosticsConfig.ActivitySource.Name)
+                    .ConfigureResource(resource => resource
+                        .AddService(diagnosticsConfig.ServiceName))
+                    .AddAspNetCoreInstrumentation()
+                    .AddSqlClientInstrumentation()
+                    .AddEntityFrameworkCoreInstrumentation()
+                    .AddHttpClientInstrumentation()
+                    .AddOtlpExporter());
+
         services.AddSingleton<IStripeAdapter, StripeAdapter>();
         services.AddSingleton<Braintree.IBraintreeGateway>((serviceProvider) =>
         {