Need help with twilio active call real time voice transcription with Twilio and .net C#

[ashish0218groot]

I am trying to transcribe real-time audio from Twilio and transcribe it using Deepgram.
While using ngrok I can get hit on my API and while my API key with owner key is being authenticated it is sending an error as the server returned status code 401 when 101 was expected. Help is much appreciated regarding this. Thanks

using Microsoft.AspNetCore.Mvc;
using Twilio.TwiML; // For VoiceResponse and TwiML elements
using Twilio; // For TwilioClient
using System.Net.Http.Headers;
using Microsoft.Extensions.Configuration;
using Twilio.TwiML.Voice;
using System.Net.WebSockets; // For WebSocket
using System.Text;
using System.Threading;
using System;
using System.Threading.Tasks; // Ensure this is included for System.Threading.Tasks.Task
using Newtonsoft.Json; // For JSON serialization

namespace TwilioDeepgramIntegration.Controllers
{
[ApiController]
[Route("api/[controller]")]
public class TwilioController : ControllerBase
{
private readonly IConfiguration _configuration;
private readonly HttpClient _httpClient;
private ClientWebSocket _webSocket; // WebSocket connection to Deepgram

    public TwilioController(IConfiguration configuration)
    {
        _configuration = configuration;

         TwilioClient.Init(
            _configuration["Twilio:AccountSid"],
            _configuration["Twilio:AuthToken"]
        );

        _httpClient = new HttpClient();
        _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", _configuration["Deepgram:ApiKey"]);
        _webSocket = new ClientWebSocket(); 
    }

    [HttpPost("voice")]
    public async System.Threading.Tasks.Task<IActionResult> ReceiveCall()
    {
        var response = new VoiceResponse();

        var start = new Start();
        start.Append(new Twilio.TwiML.Voice.Stream
        {
            Url = "https:address.ngrok-free.app", 
            Track = "both_tracks"  
        });

        response.Append(start);

        response.Say("Thank you for calling Ashish. We are now streaming the audio.");

         await ConnectToDeepgramAsync();

        return Content(response.ToString(), "application/xml");
    }

    private async System.Threading.Tasks.Task ConnectToDeepgramAsync()
    {
        try
        {
             var uri = new Uri($"wss://api.deepgram.com/v1/listen?access_token={_configuration["Deepgram:ApiKey"]}");
            Console.WriteLine($"WebSocket URL: wss://api.deepgram.com/v1/listen?access_token={_configuration["Deepgram:ApiKey"]}");


            _webSocket = new ClientWebSocket();

            // Attempt connection
            try
            {
                await _webSocket.ConnectAsync(uri, CancellationToken.None);
                Console.WriteLine("Connected to Deepgram WebSocket");
            }
            catch (WebSocketException e)
            {
                Console.WriteLine($"WebSocket Error: {e.Message}");
                throw;
            }

            await _webSocket.ConnectAsync(uri, CancellationToken.None);

             _ = System.Threading.Tasks.Task.Run(ReceiveFromDeepgramAsync); 
            _ = System.Threading.Tasks.Task.Run(SendAudioToDeepgramAsync);  
        }
        catch (Exception ex)
        {
            Console.WriteLine($"Error: {ex.Message}");
        }
    }

     private async System.Threading.Tasks.Task ReceiveFromDeepgramAsync()
    {
        while (_webSocket.State == WebSocketState.Open)
        {
            var buffer = new byte[1024 * 4];  
            var result = await _webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);

            if (result.MessageType == WebSocketMessageType.Close)
            {
                await _webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closing", CancellationToken.None);
            }
            else
            {
                 var response = Encoding.UTF8.GetString(buffer, 0, result.Count);
                var jsonResponse = JsonConvert.DeserializeObject<DeepgramResponse>(response);

                if (jsonResponse?.Channel?.Alternatives != null)
                {
                    var transcription = jsonResponse.Channel.Alternatives[0].Transcript;
                    Console.WriteLine($"Transcription: {transcription}");  
                }
            }
        }

         await System.Threading.Tasks.Task.CompletedTask;
    }

     private async System.Threading.Tasks.Task SendAudioToDeepgramAsync()
    {
        while (_webSocket.State == WebSocketState.Open)
        {
            var audioBuffer = GetAudioBuffer();  
            if (audioBuffer.Length > 0)
            {
                await _webSocket.SendAsync(new ArraySegment<byte>(audioBuffer), WebSocketMessageType.Binary, true, CancellationToken.None);
            }
        }

         await System.Threading.Tasks.Task.CompletedTask;
    }

    // Implement this method to capture audio buffer from the Twilio stream
    private byte[] GetAudioBuffer()
    {
        // Placeholder logic, replace this with actual audio capture logic
        // You might want to buffer incoming audio from Twilio here and convert it into the right format for Deepgram
        return new byte[0]; // Placeholder
    }
}

 public class DeepgramResponse
{
    public Channel Channel { get; set; } = new Channel();   
}

public class Channel
{
    public Alternative[] Alternatives { get; set; } = Array.Empty<Alternative>(); 
}

public class Alternative
{
    public string Transcript { get; set; } = string.Empty; 
}

}

[davidvonthenen]

Hi @ashish0218groot

Can you enable logging and then paste the trace here?
https://github.com/deepgram/deepgram-dotnet-sdk/blob/main/examples/speech-to-text/websocket/microphone/Program.cs#L20

[ashish0218groot]

Hi @dvonthenen
Here i have created a access key with Owner role, still this issue is occuring

Help is much appreciated.

[davidvonthenen]

Those aren't the logs that I am looking for. Can you enable the SDK logging and then paste the trace here?
https://github.com/deepgram/deepgram-dotnet-sdk/blob/main/examples/speech-to-text/websocket/microphone/Program.cs#L20

[ashish0218groot]

HI @dvonthenen here are the logs.

Here is the code:
using Microsoft.AspNetCore.Mvc;
using Twilio.TwiML;
using Twilio;
using Microsoft.Extensions.Configuration;
using Deepgram;
using Deepgram.Models.Listen.v1.WebSocket;
using System;
using System.IO;
using System.Threading.Tasks;
using Deepgram.Logger;
using Deepgram.Models.Authenticate.v1;
using LogLevel = Deepgram.Logger.LogLevel;
using Deepgram.Models.Exceptions.v1;

namespace TwilioDeepgramIntegration.Controllers
{
[ApiController]
[Route("api/[controller]")]
public class TwilioController : ControllerBase
{
private readonly IConfiguration _configuration;
private ListenWebSocketClient _liveClient;

    public TwilioController(IConfiguration configuration)
    {
        _configuration = configuration;

        TwilioClient.Init(
           _configuration["Twilio:AccountSid"],
           _configuration["Twilio:AuthToken"]
       );

        Deepgram.Library.Initialize(LogLevel.Debug);
    }

    [HttpPost("voice")]
    public async Task<IActionResult> ReceiveCall()
    {
        var response = new VoiceResponse();

         response.Append(new Twilio.TwiML.Voice.Stream
        {
            Url = "wss://key.ngrok-free.app/api/twilio/audio",
            Track = "both_tracks"
        });

        response.Say("Thank you for calling. We are now streaming the audio.");

        await ConnectToDeepgram();

        return Content(response.ToString(), "application/xml");
    }

    [HttpGet("audio")]
    public async Task GetAudioStream()
    {
        if (_liveClient == null || !_liveClient.IsConnected())
        {
            Console.WriteLine("Deepgram client is not connected.");
            return;
        }

        byte[] buffer = new byte[2048];
        int bytesRead;

        try
        {
            while ((bytesRead = await Request.Body.ReadAsync(buffer, 0, buffer.Length)) > 0)
            {
                Console.WriteLine($"Received {bytesRead} bytes of audio data.");

                _liveClient.Send(buffer[..bytesRead]);
            }
        }
        catch (Exception ex)
        {
            Console.WriteLine($"Error receiving audio stream: {ex.Message}");
        }
    }

    private async Task ConnectToDeepgram()
    {
        try
        {
            var apiKey = _configuration["Deepgram:ApiKey"];
            if (string.IsNullOrEmpty(apiKey))
            {
                throw new InvalidOperationException("Deepgram API key is not configured.");
            }

            var options = new DeepgramWsClientOptions
            {
                KeepAlive = true,
                APIVersion = "v1",
                BaseAddress = "wss://api.deepgram.com/v1"
            };

            _liveClient = new ListenWebSocketClient(apiKey, options);

             _liveClient.Subscribe((sender, e) =>
            {
                if (e.Channel?.Alternatives != null && e.Channel.Alternatives.Count > 0)
                {
                    var transcript = e.Channel.Alternatives[0].Transcript;
                    if (!string.IsNullOrWhiteSpace(transcript))
                    {
                        Console.WriteLine($"Transcription received: {transcript}");
                    }
                }
            });

            var liveSchema = new LiveSchema()
            {
                Model = "nova-2",
                Encoding = "linear16",
                SampleRate = 16000,
                Punctuate = true,
                InterimResults = true,
                UtteranceEnd = "1000",
                VadEvents = true,
            };

            // Call the Connect method (ensure it's synchronous)
            _liveClient.Connect(liveSchema);  // Remove await if Connect is not asynchronous
            Console.WriteLine("Connected to Deepgram WebSocket");
        }
        catch (DeepgramException ex)
        {
            Console.WriteLine($"Deepgram API Error: {ex.Message}");
        }
        catch (Exception ex)
        {
            Console.WriteLine($"Error connecting to Deepgram: {ex.Message}");
        }
    }
}

}

[ashish0218groot]

Hi @dvonthenen
Using DeepgramWsClientOptions i am sending keys but they are not reflecting in console. Maybe I need some changes in my approach. Can you guide me through it, Thanks


using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using System.Threading.Tasks;
using Twilio.TwiML;
using Deepgram;
using Deepgram.Models.Listen.v1.WebSocket;
using Deepgram.Models.Authenticate.v1;
using Twilio;

namespace TwilioDeepgramIntegration.Controllers
{
[ApiController]
[Route("api/[controller]")]
public class TwilioController : ControllerBase
{
private readonly IConfiguration _configuration;
private ListenWebSocketClient _liveClient;
private readonly ILogger _logger;

    public TwilioController(IConfiguration configuration, ILogger<TwilioController> logger)
    {
        _configuration = configuration;
        _logger = logger;

        // Initialize Twilio Client
        TwilioClient.Init(_configuration["Twilio:AccountSid"], _configuration["Twilio:AuthToken"]);

        // Initialize Deepgram Library
        Deepgram.Library.Initialize();
    }

    [HttpPost("voice")]
    public IActionResult ReceiveCall()
    {
        _logger.LogInformation("Received an incoming call from Twilio.");
        var response = new VoiceResponse();
        var audioStreamUrl = "wss://key.ngrok-free.app/api/twilio/audio";

        response.Append(new Twilio.TwiML.Voice.Stream
        {
            Url = audioStreamUrl,
            Track = "inbound"
        });
        response.Say("Thank you for calling. We are now streaming the audio.");

        // Use Task.Run to not block the main thread
        Task.Run(() => ConnectToDeepgram());

        return Content(response.ToString(), "application/xml");
    }

    [HttpPost("audio")]
    public async Task<IActionResult> GetAudioStream()
    {
        if (_liveClient == null || !_liveClient.IsConnected())
        {
            _logger.LogWarning("Deepgram client is not connected.");
            return BadRequest("Deepgram client not connected.");
        }

        var buffer = new byte[4096];
        int bytesRead;

        _logger.LogInformation("Reading audio stream from Twilio...");

        try
        {
            while ((bytesRead = await Request.Body.ReadAsync(buffer, 0, buffer.Length)) > 0)
            {
                _liveClient.Send(buffer.AsSpan(0, bytesRead).ToArray());
                _logger.LogInformation($"Sent {bytesRead} bytes to Deepgram.");
            }

            return Ok("Audio streaming to Deepgram.");
        }
        catch (Exception ex)
        {
            _logger.LogError($"Streaming error: {ex.Message}");
            return StatusCode(500, "Error streaming audio.");
        }
    }

    private async Task ConnectToDeepgram()
    {
        _logger.LogInformation("Connecting to Deepgram WebSocket...");
        var apiKey = _configuration["Deepgram:ApiKey"];
        var options = new DeepgramWsClientOptions
        {
            ApiKey = apiKey,
            BaseAddress = _configuration["Deepgram:BaseAddress"],
            KeepAlive = true // Keep the connection alive
        };

        _liveClient = new ListenWebSocketClient(apiKey, options);

        var liveSchema = new LiveSchema
        {
            Model = "nova-2",
            Encoding = "linear16",
            SampleRate = 16000,
            Punctuate = true,
            InterimResults = true
        };

        try
        {
            await _liveClient.Connect(liveSchema);
            _logger.LogInformation("Connected to Deepgram WebSocket");

            _liveClient.Subscribe((sender, e) =>
            {
                var transcript = e.Channel?.Alternatives?[0]?.Transcript;
                if (!string.IsNullOrWhiteSpace(transcript))
                {
                    _logger.LogInformation($"Transcription received: {transcript}");
                }
            });
        }
        catch (Exception ex)
        {
            _logger.LogError($"Error connecting to Deepgram: {ex.Message}");
        }
    }
}

}