fix: Improve error handling to avoid UnknownErrors (#1231)

README.md

@@ -220,6 +220,14 @@ This is the list of operational codes that can help you understand your deployme
 | UnableToUpdateCounter              | Error when trying to update a counter to track rate limits for a tenant                                                             |
 | UnableToFindCounter                | Error when trying to find a counter to track rate limits for a tenant                                                               |
 | UnhandledProcessMessage            | Unhandled message received by a Realtime process                                                                                    |
+| UnableToSetPolicies                | We were not able to set policies for this connection                                                                                |
+| ConnectionInitializing             | Database is initializing connection                                                                                                 |
+| DatabaseConnectionIssue            | Database had connection issues and connection was not able to be established                                                        |
+| UnableToConnectToProject           | Unable to connect to Project database                                                                                               |
+| InvalidJWTExpiration               | JWT exp claim value it's incorrect                                                                                                  |
+| JwtSignatureError                  | JWT signature was not able to be validated                                                                                          |
+| Unauthorized                       | Unauthorized access to Realtime channel                                                                                             |
+| RealtimeRestarting                 | Realtime is currently restarting                                                                                                    |
 | UnableToProcessListenPayload       | Payload sent in NOTIFY operation was JSON parsable                                                                                  |
 | UnableToListenToTenantDatabase     | Unable to LISTEN for notifications against the Tenant Database                                                                      |
 | UnprocessableEntity                | Received a HTTP request with a body that was not able to be processed by the endpoint                                               |

lib/realtime/signal_handler.ex

@@ -3,9 +3,12 @@ defmodule Realtime.SignalHandler do
   @behaviour :gen_event
   require Logger
 
-  @spec shutdown_in_progress? :: boolean()
+  @spec shutdown_in_progress? :: :ok | {:error, :shutdown_in_progress}
   def shutdown_in_progress? do
-    !!Application.get_env(:realtime, :shutdown_in_progress)
+    case !!Application.get_env(:realtime, :shutdown_in_progress) do
+      true -> {:error, :shutdown_in_progress}
+      false -> :ok
+    end
   end
 
   @impl true

lib/realtime_web/channels/realtime_channel.ex

@@ -49,7 +49,7 @@ defmodule RealtimeWeb.RealtimeChannel do
 
     start_db_rate_counter(tenant_id)
 
-    with false <- SignalHandler.shutdown_in_progress?(),
+    with :ok <- SignalHandler.shutdown_in_progress?(),
          :ok <- only_private?(tenant_id, socket),
          :ok <- limit_joins(socket.assigns),
          :ok <- limit_channels(socket),
@@ -126,6 +126,13 @@ defmodule RealtimeWeb.RealtimeChannel do
           "Realtime is initializing the project connection"
         )
 
+      {:error, :tenant_database_connection_initializing} ->
+        Logging.log_error_message(
+          :warning,
+          "InitializingProjectConnection",
+          "Connecting to the project database"
+        )
+
       {:error, invalid_exp} when is_integer(invalid_exp) and invalid_exp <= 0 ->
         Logging.log_error_message(
           :error,
@@ -140,6 +147,32 @@ defmodule RealtimeWeb.RealtimeChannel do
           "This project only allows private channels"
         )
 
+      {:error, :signature_error} ->
+        Logging.log_error_message(:error, "JwtSignatureError", "Failed to validate JWT signature")
+
+      {:error, %Postgrex.Error{} = error} ->
+        Logging.log_error_message(:error, "DatabaseConnectionIssue", error)
+
+      {:error, %DBConnection.ConnectionError{} = error} ->
+        Logging.log_error_message(:error, "DatabaseConnectionIssue", error)
+
+      {:error, :shutdown_in_progress} ->
+        Logging.log_error_message(
+          :error,
+          "RealtimeRestarting",
+          "Realtime is restarting, please standby"
+        )
+
+      {:error, :unable_to_set_policies} ->
+        Logging.log_error_message(
+          :error,
+          "UnableToSetPolicies",
+          "Unable to set policies for connection"
+        )
+
+      {:error, :unauthorized, msg} ->
+        Logging.log_error_message(:error, "Unauthorized", msg)
+
       {:error, error} ->
         Logging.log_error_message(:error, "UnknownErrorOnChannel", error)
 
@@ -349,15 +382,18 @@ defmodule RealtimeWeb.RealtimeChannel do
 
       {:noreply, assign(socket, assigns)}
     else
+      {:error, :unauthorized, msg} ->
+        shutdown_response(socket, msg)
+
       {:error, error} when is_binary(error) ->
-        message = "Received an invalid access token from client: " <> error
+        msg = "Received an invalid access token from client: " <> error
 
-        shutdown_response(socket, message)
+        shutdown_response(socket, msg)
 
       {:error, error} ->
-        message = "Received an invalid access token from client: " <> inspect(error)
+        msg = "Received an invalid access token from client: " <> inspect(error)
 
-        shutdown_response(socket, message)
+        shutdown_response(socket, msg)
     end
   end
 
@@ -670,15 +706,21 @@ defmodule RealtimeWeb.RealtimeChannel do
     with {:ok, socket} <- Authorization.get_authorizations(socket, db_conn, authorization_context) do
       cond do
         match?(%Policies{broadcast: %BroadcastPolicies{read: false}}, socket.assigns.policies) ->
-          {:error, "You do not have permissions to read from this Channel topic: #{topic}"}
+          {:error, :unauthorized,
+           "You do not have permissions to read from this Channel topic: #{topic}"}
 
         using_broadcast? and
             match?(%Policies{broadcast: %BroadcastPolicies{read: false}}, socket.assigns.policies) ->
-          {:error, "You do not have permissions to read from this Channel topic: #{topic}"}
+          {:error, :unauthorized,
+           "You do not have permissions to read from this Channel topic: #{topic}"}
 
         true ->
           {:ok, socket}
       end
+    else
+      {:error, error} ->
+        log_error("UnableToSetPolicies", error)
+        {:error, :unable_to_set_policies}
     end
   end
 

mix.exs

@@ -4,7 +4,7 @@ defmodule Realtime.MixProject do
   def project do
     [
       app: :realtime,
-      version: "2.33.51",
+      version: "2.33.52",
       elixir: "~> 1.16.0",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,

test/integration/rt_channel_test.exs

@@ -547,7 +547,7 @@ defmodule Realtime.Integration.RtChannelTest do
       })
 
       error_message =
-        "Received an invalid access token from client: You do not have permissions to read from this Channel topic: #{topic}"
+        "You do not have permissions to read from this Channel topic: #{topic}"
 
       assert_receive %Phoenix.Socket.Message{
         event: "system",