Add retry capability to REST client to handle transient errors

src/Microsoft.Azure.SignalR.Common/Constants.cs

@@ -113,5 +113,11 @@ public static class ErrorCodes
  public const string InfoUserNotInGroup = "Info.User.NotInGroup";
  public const string ErrorConnectionNotExisted = "Error.Connection.NotExisted";
  }
+
+ public static class HttpClientNames
+ {
+ public const string Resilient = "Resilient";
+ public const string MessageResilient = "MessageResilient";
+ }
  }
 }

src/Microsoft.Azure.SignalR.Common/Utilities/RestClient.cs

@@ -11,8 +11,11 @@
 using System.Threading.Tasks;
 using Azure.Core.Serialization;
 using Microsoft.Azure.SignalR.Common;
+using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Primitives;
 
+#nullable enable
+
 namespace Microsoft.Azure.SignalR
 {
  internal class RestClient
@@ -28,7 +31,6 @@ public RestClient(IHttpClientFactory httpClientFactory, IPayloadContentBuilder c
  _enableMessageTracing = enableMessageTracing;
  }
 
-
  public RestClient(IHttpClientFactory httpClientFactory, ObjectSerializer objectSerializer, bool enableMessageTracing) : this(httpClientFactory, new JsonPayloadContentBuilder(objectSerializer), enableMessageTracing)
  {
  }
@@ -42,9 +44,9 @@ public Task SendAsync(
  RestApiEndpoint api,
  HttpMethod httpMethod,
  string productInfo,
- string methodName = null,
- object[] args = null,
- Func<HttpResponseMessage, bool> handleExpectedResponse = null,
+ string? methodName = null,
+ object[]? args = null,
+ Func<HttpResponseMessage, bool>? handleExpectedResponse = null,
  CancellationToken cancellationToken = default)
  {
  if (handleExpectedResponse == null)
@@ -55,40 +57,43 @@ public Task SendAsync(
  return SendAsync(api, httpMethod, productInfo, methodName, args, response => Task.FromResult(handleExpectedResponse(response)), cancellationToken);
  }
 
- public async Task SendAsync(
+ public Task SendAsync(
  RestApiEndpoint api,
  HttpMethod httpMethod,
  string productInfo,
- string methodName = null,
- object[] args = null,
- Func<HttpResponseMessage, Task<bool>> handleExpectedResponseAsync = null,
+ string? methodName = null,
+ object[]? args = null,
+ Func<HttpResponseMessage, Task<bool>>? handleExpectedResponseAsync = null,
  CancellationToken cancellationToken = default)
  {
- using var httpClient = _httpClientFactory.CreateClient();
-  using var request = BuildRequest(api, httpMethod, productInfo, methodName, args);
+ return SendAsyncCore(Options.DefaultName, api, httpMethod, productInfo, methodName, args, handleExpectedResponseAsync, cancellationToken);
+ }
 
- try
- {
- using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
- if (handleExpectedResponseAsync == null)
- {
- await ThrowExceptionOnResponseFailureAsync(response);
- }
- else
- {
- if (!await handleExpectedResponseAsync(response))
- {
- await ThrowExceptionOnResponseFailureAsync(response);
- }
- }
- }
- catch (HttpRequestException ex)
- {
- throw new AzureSignalRException($"An error happened when making request to {request.RequestUri}", ex);
- }
+ public Task SendWithRetryAsync(
+ RestApiEndpoint api,
+ HttpMethod httpMethod,
+ string productInfo,
+ string? methodName = null,
+ object[]? args = null,
+ Func<HttpResponseMessage, bool>? handleExpectedResponse = null,
+ CancellationToken cancellationToken = default)
+ {
+ return SendAsyncCore(Constants.HttpClientNames.Resilient, api, httpMethod, productInfo, methodName, args, handleExpectedResponse == null ? null : response => Task.FromResult(handleExpectedResponse(response)), cancellationToken);
+ }
+
+ public Task SendMessageWithRetryAsync(
+ RestApiEndpoint api,
+ HttpMethod httpMethod,
+ string productInfo,
+ string? methodName = null,
+ object[]? args = null,
+ Func<HttpResponseMessage, bool>? handleExpectedResponse = null,
+ CancellationToken cancellationToken = default)
+ {
+ return SendAsyncCore(Constants.HttpClientNames.MessageResilient, api, httpMethod, productInfo, methodName, args, handleExpectedResponse == null ? null : response => Task.FromResult(handleExpectedResponse(response)), cancellationToken);
  }
 
- public async Task ThrowExceptionOnResponseFailureAsync(HttpResponseMessage response)
+ private async Task ThrowExceptionOnResponseFailureAsync(HttpResponseMessage response)
  {
  if (response.IsSuccessStatusCode)
  {
@@ -106,14 +111,48 @@ public async Task ThrowExceptionOnResponseFailureAsync(HttpResponseMessage respo
 #endif
  throw response.StatusCode switch
  {
- HttpStatusCode.BadRequest => new AzureSignalRInvalidArgumentException(response.RequestMessage.RequestUri.ToString(), innerException, detail),
- HttpStatusCode.Unauthorized => new AzureSignalRUnauthorizedException(response.RequestMessage.RequestUri.ToString(), innerException),
- HttpStatusCode.NotFound => new AzureSignalRInaccessibleEndpointException(response.RequestMessage.RequestUri.ToString(), innerException),
- _ => new AzureSignalRRuntimeException(response.RequestMessage.RequestUri.ToString(), innerException),
+ HttpStatusCode.BadRequest => new AzureSignalRInvalidArgumentException(response.RequestMessage?.RequestUri?.ToString(), innerException, detail),
+ HttpStatusCode.Unauthorized => new AzureSignalRUnauthorizedException(response.RequestMessage?.RequestUri?.ToString(), innerException),
+ HttpStatusCode.NotFound => new AzureSignalRInaccessibleEndpointException(response.RequestMessage?.RequestUri?.ToString(), innerException),
+ _ => new AzureSignalRRuntimeException(response.RequestMessage?.RequestUri?.ToString(), innerException),
  };
  }
 
- private static Uri GetUri(string url, IDictionary<string, StringValues> query)
+ private async Task SendAsyncCore(
+ string httpClientName,
+ RestApiEndpoint api,
+ HttpMethod httpMethod,
+ string productInfo,
+ string? methodName = null,
+ object[]? args = null,
+ Func<HttpResponseMessage, Task<bool>>? handleExpectedResponseAsync = null,
+ CancellationToken cancellationToken = default)
+ {
+ using var httpClient = _httpClientFactory.CreateClient(httpClientName);
+ using var request = BuildRequest(api, httpMethod, productInfo, methodName, args);
+
+ try
+ {
+ using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+ if (handleExpectedResponseAsync == null)
+ {
+ await ThrowExceptionOnResponseFailureAsync(response);
+ }
+ else
+ {
+ if (!await handleExpectedResponseAsync(response))
+ {
+ await ThrowExceptionOnResponseFailureAsync(response);
+ }
+ }
+ }
+ catch (HttpRequestException ex)
+ {
+ throw new AzureSignalRException($"An error happened when making request to {request.RequestUri}", ex);
+ }
+ }
+
+ private static Uri GetUri(string url, IDictionary<string, StringValues>? query)
  {
  if (query == null || query.Count == 0)
  {
@@ -136,14 +175,14 @@ private static Uri GetUri(string url, IDictionary<string, StringValues> query)
  sb.Append(sb.Length > 0 ? '&' : '?');
  sb.Append(Uri.EscapeDataString(item.Key));
  sb.Append('=');
- sb.Append(Uri.EscapeDataString(value));
+ sb.Append(Uri.EscapeDataString(value!));
  }
  }
  builder.Query = sb.ToString();
  return builder.Uri;
  }
 
- private HttpRequestMessage BuildRequest(RestApiEndpoint api, HttpMethod httpMethod, string productInfo, string methodName = null, object[] args = null)
+ private HttpRequestMessage BuildRequest(RestApiEndpoint api, HttpMethod httpMethod, string productInfo, string? methodName = null, object[]? args = null)
  {
  var payload = httpMethod == HttpMethod.Post ? new PayloadMessage { Target = methodName, Arguments = args } : null;
  if (_enableMessageTracing)
@@ -153,7 +192,7 @@ private HttpRequestMessage BuildRequest(RestApiEndpoint api, HttpMethod httpMeth
  return GenerateHttpRequest(api.Audience, api.Query, httpMethod, payload, api.Token, productInfo);
  }
 
- private HttpRequestMessage GenerateHttpRequest(string url, IDictionary<string, StringValues> query, HttpMethod httpMethod, PayloadMessage payload, string tokenString, string productInfo)
+ private HttpRequestMessage GenerateHttpRequest(string url, IDictionary<string, StringValues> query, HttpMethod httpMethod, PayloadMessage? payload, string tokenString, string productInfo)
  {
  var request = new HttpRequestMessage(httpMethod, GetUri(url, query));
  request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", tokenString);

src/Microsoft.Azure.SignalR.Management/Configuration/ServiceManagerOptions.cs

@@ -6,6 +6,8 @@
 using Azure.Core.Serialization;
 using Newtonsoft.Json;
 
+#nullable enable
+
 namespace Microsoft.Azure.SignalR.Management
 {
  /// <summary>
@@ -16,7 +18,7 @@ public class ServiceManagerOptions
  /// <summary>
  /// Gets or sets the ApplicationName which will be prefixed to each hub name
  /// </summary>
- public string ApplicationName { get; set; }
+ public string? ApplicationName { get; set; }
 
  /// <summary>
  /// Gets or sets the total number of connections from SDK to Azure SignalR Service. Default value is 1.
@@ -26,17 +28,17 @@ public class ServiceManagerOptions
  /// <summary>
  /// Gets or sets a service endpoint of Azure SignalR Service instance by connection string.
  /// </summary>
- public string ConnectionString { get; set; } = null;
+ public string? ConnectionString { get; set; } = null;
 
  /// <summary>
  /// Gets or sets multiple service endpoints of Azure SignalR Service instances.
  /// </summary>
- public ServiceEndpoint[] ServiceEndpoints { get; set; }
+ public ServiceEndpoint[]? ServiceEndpoints { get; set; }
 
  /// <summary>
  /// Gets or sets the proxy used when ServiceManager will attempt to connect to Azure SignalR Service.
  /// </summary>
- public IWebProxy Proxy { get; set; }
+ public IWebProxy? Proxy { get; set; }
 
  /// <summary>
  /// Gets or sets the transport type to Azure SignalR Service. Default value is Transient.
@@ -48,6 +50,8 @@ public class ServiceManagerOptions
  /// </summary>
  public TimeSpan HttpClientTimeout { get; set; } = TimeSpan.FromSeconds(100);
 
+ public ServiceManagerRetryOptions? RetryOptions { get; set; }
+
  /// <summary>
  /// Gets the json serializer settings that will be used to serialize content sent to Azure SignalR Service.
  /// </summary>
@@ -57,7 +61,7 @@ public class ServiceManagerOptions
  /// <summary>
  /// If users want to use MessagePack, they should go to <see cref="ServiceManagerBuilder.AddHubProtocol(AspNetCore.SignalR.Protocol.IHubProtocol)"/>
  /// </summary>
- internal ObjectSerializer ObjectSerializer { get; set; }
+ internal ObjectSerializer? ObjectSerializer { get; set; }
 
  /// <summary>
  /// Set a JSON object serializer used to serialize the data sent to clients.
@@ -73,7 +77,7 @@ public void UseJsonObjectSerializer(ObjectSerializer objectSerializer)
  // not ready
  internal bool EnableMessageTracing { get; set; } = false;
 
- internal string ProductInfo { get; set; }
+ internal string? ProductInfo { get; set; }
 
  internal void ValidateOptions()
  {

src/Microsoft.Azure.SignalR.Management/Configuration/ServiceManagerRetryMode.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.Azure.SignalR.Management;
+
+#nullable enable
+
+/// <summary>
+/// The type of approach to apply when calculating the delay between retry attempts.
+/// </summary>
+public enum ServiceManagerRetryMode
+{
+ /// <summary>
+ /// Retry attempts happen at fixed intervals; each delay is a consistent duration.
+ /// </summary>
+ Fixed,
+ /// <summary>
+ /// Retry attempts will delay based on a backoff strategy, where each attempt will
+ /// increase the duration that it waits before retrying.
+ /// </summary>
+ Exponential
+}

src/Microsoft.Azure.SignalR.Management/Configuration/ServiceManagerRetryOptions.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+
+namespace Microsoft.Azure.SignalR.Management;
+
+#nullable enable
+
+public class ServiceManagerRetryOptions
+{
+ /// <summary>
+ /// The maximum number of retry attempts before giving up.
+ /// </summary>
+ public int MaxRetries { get; set; } = 3;
+
+ /// <summary>
+ /// The delay between retry attempts for a fixed approach or the delay
+ /// on which to base calculations for a backoff-based approach.
+ /// </summary>
+ public TimeSpan Delay { get; set; } = TimeSpan.FromSeconds(0.8);
+
+ /// <summary>
+ /// The maximum permissible delay between retry attempts.
+ /// </summary>
+ public TimeSpan MaxDelay { get; set; } = TimeSpan.FromMinutes(1);
+
+ /// <summary>
+ /// The approach to use for calculating retry delays.
+ /// </summary>
+ public ServiceManagerRetryMode Mode { get; set; } = ServiceManagerRetryMode.Fixed;
+}
+
+