wip

examples/demo.exs

@@ -31,7 +31,7 @@ defmodule OpenAIElement do
  require Membrane.Logger
 
  def_input_pad :input, accepted_format: _any
- # def_output_pad :output, accepted_format: _any
+ def_output_pad :output, accepted_format: _any, flow_control: :push
 
  def_options websocket_opts: []
 
@@ -41,6 +41,12 @@ defmodule OpenAIElement do
  {[], %{ws: ws}}
  end
 
+ @impl true
+ def handle_playing(_ctx, state) do
+ format = %Membrane.RawAudio{channels: 1, sample_rate: 24_000, sample_format: :s16le}
+ {[stream_format: {:output, format}], state}
+ end
+
  @impl true
  def handle_buffer(:input, buffer, _ctx, state) do
  # buffer preprocessing
@@ -71,14 +77,15 @@ defmodule OpenAIElement do
  defp create_response(ws) do
  # frame = Jason.encode(%{type: "input_audio_buffer.commit"})
  frame = Jason.encode(%{type: "response.create"})
- :ok = OpenAIWebSocket.send_frame(state.ws, frame)
+ :ok = OpenAIWebSocket.send_frame(ws, frame)
  end
 
  @impl true
  def handle_info({:websocket_frame, frame}, _ctx, state) do
  decoded_frame = Jason.decode!(frame)
  dbg(decoded_frame, label: "DECODED FRAME")
- {[], state}
+ # {[], state}
+ {[buffer: {:output, %Membrane.Buffer{payload: decoded_frame.audio}}], state}
  end
 end
 
@@ -89,16 +96,22 @@ defmodule BrowserToOpenAi do
  def handle_init(_ctx, opts) do
  spec =
  child(:webrtc_source, %Membrane.WebRTC.Source{
- signaling: {:websocket, port: opts[:webrtc_ws_port]}
+ signaling: {:websocket, port: opts[:webrtc_source_ws_port]}
  })
  |> via_out(:output, options: [kind: :audio])
  |> child(:opus_parser, Membrane.Opus.Parser)
  |> child(:opus_decoder, %Membrane.Opus.Decoder{sample_rate: 24_000})
  |> child(:open_ai, %OpenAIElement{websocket_opts: opts[:openai_ws_opts]})
+ |> child(:raw_audio_parser, Membrane.RawAudioParser)
+ |> child(:opus_encoder, Membrane.Opus.Encoder)
+ |> via_in(:input, options: [kind: :audio])
+ |> child(:webrtc_sink, %Membrane.WebRTC.Sink{
+ signaling: {:websocket, port: opts[:webrtc_source_ws_port]}
+ })
 
  self() |> Process.send_after(:create_response, 10_000)
 
- {[spec: spec], state}
+ {[spec: spec], %{}}
  end
 
  @impl true

mix.exs

@@ -39,6 +39,8 @@ defmodule Membrane.Template.Mixfile do
  [
  {:membrane_core, "~> 1.1"},
  {:membrane_webrtc_plugin, "~> 0.22.0"},
+ {:membrane_opus_plugin, "~> 0.20.3"},
+ {:membrane_raw_audio_parser_plugin, "~> 0.4.0" },
  {:bandit, "~> 1.2"},
  {:websock_adapter, "~> 0.5.0"},
  {:websockex, "~> 0.4.3"},

mix.lock

@@ -31,11 +31,15 @@
  "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"},
  "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"},
  "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"},
+ "membrane_common_c": {:hex, :membrane_common_c, "0.16.0", "caf3f29d2f5a1d32d8c2c122866110775866db2726e4272be58e66dfdf4bce40", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:shmex, "~> 0.5.0", [hex: :shmex, repo: "hexpm", optional: false]}, {:unifex, "~> 1.0", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "a3c7e91de1ce1f8b23b9823188a5d13654d317235ea0ca781c05353ed3be9b1c"},
  "membrane_core": {:hex, :membrane_core, "1.1.1", "4dcff6e9f3b2ecd4f437c20e201e53957731772c0f15b3005062c41f7f58f500", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:qex, "~> 0.3", [hex: :qex, repo: "hexpm", optional: false]}, {:ratio, "~> 3.0 or ~> 4.0", [hex: :ratio, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3802f3fc071505c59d48792487d9927e803d4edb4039710ffa52cdb60bb0aecc"},
  "membrane_funnel_plugin": {:hex, :membrane_funnel_plugin, "0.9.1", "9e108f4ef9d905ebff2da3ba5e58a5b756b58812f4fa68bd576add68fda310a0", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "39fdef1bf29eac949f65a37ea941f997c22ed042c55af044d27a781b63e82f6b"},
  "membrane_h264_format": {:hex, :membrane_h264_format, "0.6.1", "44836cd9de0abe989b146df1e114507787efc0cf0da2368f17a10c47b4e0738c", [:mix], [], "hexpm", "4b79be56465a876d2eac2c3af99e115374bbdc03eb1dea4f696ee9a8033cd4b0"},
  "membrane_opus_format": {:hex, :membrane_opus_format, "0.3.0", "3804d9916058b7cfa2baa0131a644d8186198d64f52d592ae09e0942513cb4c2", [:mix], [], "hexpm", "8fc89c97be50de23ded15f2050fe603dcce732566fe6fdd15a2de01cb6b81afe"},
+ "membrane_opus_plugin": {:hex, :membrane_opus_plugin, "0.20.3", "871b8e3fcc1ea16e9f35a45e7a7938dda376653de73837c00c58ef416b69540f", [:mix], [{:bunch, "~> 1.3", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.2", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_opus_format, "~> 0.3.0", [hex: :membrane_opus_format, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_audio_format, "~> 0.12.0", [hex: :membrane_raw_audio_format, repo: "hexpm", optional: false]}, {:unifex, "~> 1.0", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "d5acd9359ff38f91a74ed06417396a0df10213574f01058506d7462233642125"},
  "membrane_precompiled_dependency_provider": {:hex, :membrane_precompiled_dependency_provider, "0.1.2", "8af73b7dc15ba55c9f5fbfc0453d4a8edfb007ade54b56c37d626be0d1189aba", [:mix], [{:bundlex, "~> 1.4", [hex: :bundlex, repo: "hexpm", optional: false]}], "hexpm", "7fe3e07361510445a29bee95336adde667c4162b76b7f4c8af3aeb3415292023"},
+ "membrane_raw_audio_format": {:hex, :membrane_raw_audio_format, "0.12.0", "b574cd90f69ce2a8b6201b0ccf0826ca28b0fbc8245b8078d9f11cef65f7d5d5", [:mix], [{:bimap, "~> 1.1", [hex: :bimap, repo: "hexpm", optional: false]}, {:bunch, "~> 1.0", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "6e6c98e3622a2b9df19eab50ba65d7eb45949b1ba306fa8423df6cdb12fd0b44"},
+ "membrane_raw_audio_parser_plugin": {:hex, :membrane_raw_audio_parser_plugin, "0.4.0", "7a1e53b68a221d00e47fb5d3c7e29200dfe8f7bc0862e69000b61c6562093acc", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_raw_audio_format, "~> 0.12.0", [hex: :membrane_raw_audio_format, repo: "hexpm", optional: false]}], "hexpm", "ff8d3fba45b1c2814b68d49878f19d2c1ad1147b53f606b48b6b67068435dcd0"},
  "membrane_rtp_format": {:hex, :membrane_rtp_format, "0.8.0", "828924bbd27efcf85b2015ae781e824c4a9928f0a7dc132abc66817b2c6edfc4", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "bc75d2a649dfaef6df563212fbb9f9f62eebc871393692f9dae8d289bd4f94bb"},
  "membrane_rtp_h264_plugin": {:hex, :membrane_rtp_h264_plugin, "0.19.3", "4c5c4e17e2544897a8436432753a858c8759f3bc7e0e58c033f131d959caf6a8", [:mix], [{:bunch, "~> 1.5", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_h264_format, "~> 0.6.0", [hex: :membrane_h264_format, repo: "hexpm", optional: false]}, {:membrane_rtp_format, "~> 0.8.0", [hex: :membrane_rtp_format, repo: "hexpm", optional: false]}], "hexpm", "778bab15fa45dfb37cf416106157f8adb502555eb30d80e73ba26bf58ec1d888"},
  "membrane_rtp_opus_plugin": {:hex, :membrane_rtp_opus_plugin, "0.9.0", "ae76421faa04697a4af76a55b6c5e675dea61b611d29d8201098783d42863af7", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_opus_format, "~> 0.3.0", [hex: :membrane_opus_format, repo: "hexpm", optional: false]}, {:membrane_rtp_format, "~> 0.8.0", [hex: :membrane_rtp_format, repo: "hexpm", optional: false]}], "hexpm", "58f095d2978daf999d87c1c016007cb7d99434208486331ab5045e77f5be9dcc"},