From d287c95d896c4f3177979a1c9398ad0183e94c6a Mon Sep 17 00:00:00 2001 From: Philip Giuliani Date: Mon, 30 Sep 2024 10:07:41 +0200 Subject: [PATCH] Add fillers --- lib/membrane/hls/filler/aac_filler.ex | 56 +++++++++++++++++++ lib/membrane/hls/filler/text_filler.ex | 39 +++++++++++++ .../hls/{text_filler.ex => shifter.ex} | 40 ------------- lib/membrane/hls/sink_bin.ex | 17 +++++- test/membrane/hls/sink_bin_test.exs | 6 +- 5 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 lib/membrane/hls/filler/aac_filler.ex create mode 100644 lib/membrane/hls/filler/text_filler.ex rename lib/membrane/hls/{text_filler.ex => shifter.ex} (50%) diff --git a/lib/membrane/hls/filler/aac_filler.ex b/lib/membrane/hls/filler/aac_filler.ex new file mode 100644 index 0000000..4a08d18 --- /dev/null +++ b/lib/membrane/hls/filler/aac_filler.ex @@ -0,0 +1,56 @@ +defmodule Membrane.HLS.AACFiller do + use Membrane.Filter + + def_input_pad(:input, + accepted_format: Membrane.AAC + ) + + def_output_pad(:output, + accepted_format: Membrane.RemoteStream + ) + + def_options( + duration: [ + spec: Membrane.Time.t() + ] + ) + + def handle_init(_ctx, opts) do + {[], %{duration: opts.duration, format: nil, filled: false}} + end + + def handle_stream_format(:input, format, _ctx, state) do + {[forward: %Membrane.RemoteStream{content_format: Membrane.AAC}], %{state | format: format}} + end + + def handle_buffer(:input, buffer, _ctx, state) do + buffer = %{buffer | pts: nil, dts: nil} + + if state.filled do + {[forward: buffer], state} + else + format = state.format + + silence_buffer = + if Membrane.Time.as_milliseconds(state.duration, :round) > 0 do + duration = + Membrane.Time.as_seconds(state.duration) + |> Ratio.to_float() + + {silence, 0} = + System.cmd( + "ffmpeg", + ~w(-f lavfi -i anullsrc=r=#{format.sample_rate} -ac #{format.channels} -t #{duration} -c:a aac -f adts -) + ) + + %Membrane.Buffer{ + payload: silence + } + else + nil + end + + {[buffer: {:output, List.wrap(silence_buffer) ++ [buffer]}], %{state | filled: true}} + end + end +end diff --git a/lib/membrane/hls/filler/text_filler.ex b/lib/membrane/hls/filler/text_filler.ex new file mode 100644 index 0000000..dae49d6 --- /dev/null +++ b/lib/membrane/hls/filler/text_filler.ex @@ -0,0 +1,39 @@ +defmodule Membrane.HLS.TextFiller do + use Membrane.Filter + + def_input_pad(:input, + accepted_format: Membrane.Text + ) + + def_output_pad(:output, + accepted_format: Membrane.Text + ) + + def_options( + from: [ + spec: Membrane.Time.t() + ] + ) + + def handle_init(_ctx, opts) do + {[], %{from: opts.from, filled: false}} + end + + def handle_buffer(:input, buffer, _ctx, state) do + if state.filled do + {[forward: buffer], state} + else + Membrane.Logger.debug( + "Generated empty text buffer with a duration of #{buffer.pts - state.from - Membrane.Time.millisecond()}" + ) + + silence_buffer = %Membrane.Buffer{ + payload: "", + pts: state.from, + metadata: %{to: buffer.pts - Membrane.Time.millisecond()} + } + + {[buffer: {:output, [silence_buffer, buffer]}], %{state | filled: true}} + end + end +end diff --git a/lib/membrane/hls/text_filler.ex b/lib/membrane/hls/shifter.ex similarity index 50% rename from lib/membrane/hls/text_filler.ex rename to lib/membrane/hls/shifter.ex index 8b8d54e..546fedd 100644 --- a/lib/membrane/hls/text_filler.ex +++ b/lib/membrane/hls/shifter.ex @@ -1,43 +1,3 @@ -defmodule Membrane.HLS.TextFiller do - use Membrane.Filter - - def_input_pad(:input, - accepted_format: Membrane.Text - ) - - def_output_pad(:output, - accepted_format: Membrane.Text - ) - - def_options( - from: [ - spec: Membrane.Time.t() - ] - ) - - def handle_init(_ctx, opts) do - {[], %{from: opts.from, filled: false}} - end - - def handle_buffer(:input, buffer, _ctx, state) do - if state.filled do - {[forward: buffer], state} - else - Membrane.Logger.debug( - "Generated empty text buffer with a duration of #{buffer.pts - state.from - Membrane.Time.millisecond()}" - ) - - silence_buffer = %Membrane.Buffer{ - payload: "", - pts: state.from, - metadata: %{to: buffer.pts - Membrane.Time.millisecond()} - } - - {[buffer: {:output, [silence_buffer, buffer]}], %{state | filled: true}} - end - end -end - defmodule Membrane.HLS.Shifter do use Membrane.Filter diff --git a/lib/membrane/hls/sink_bin.ex b/lib/membrane/hls/sink_bin.ex index b07ebb5..e8dd174 100644 --- a/lib/membrane/hls/sink_bin.ex +++ b/lib/membrane/hls/sink_bin.ex @@ -108,12 +108,23 @@ defmodule Membrane.HLS.SinkBin do state ) when encoding in [:H264, :AAC] do - {max_pts, _track_pts} = resume_info(state.packager_pid, track_id) + {max_pts, track_pts} = resume_info(state.packager_pid, track_id) spec = bin_input(pad) - |> child({:shifter, track_id}, %Membrane.HLS.Shifter{duration: max_pts}) - # |> child({:filler, track_id}, %Membrane.HLS.AACFiller{from: track_pts}) + |> then(fn spec -> + if encoding == :AAC do + spec + |> child({:filler, track_id}, %Membrane.HLS.AACFiller{duration: max_pts - track_pts}) + |> child(:fix_parser, %Membrane.AAC.Parser{ + out_encapsulation: :none, + output_config: :esds + }) + |> child({:shifter, track_id}, %Membrane.HLS.Shifter{duration: track_pts}) + else + child(spec, {:shifter, track_id}, %Membrane.HLS.Shifter{duration: max_pts}) + end + end) |> child({:muxer, track_id}, %Membrane.MP4.Muxer.CMAF{ segment_min_duration: state.opts.min_segment_duration }) diff --git a/test/membrane/hls/sink_bin_test.exs b/test/membrane/hls/sink_bin_test.exs index 7ca9228..e72f672 100644 --- a/test/membrane/hls/sink_bin_test.exs +++ b/test/membrane/hls/sink_bin_test.exs @@ -8,7 +8,7 @@ defmodule Membrane.HLS.SinkBinTest do test "on a new stream", %{tmp_dir: tmp_dir} do spec = [ child(:sink, %Membrane.HLS.SinkBin{ - manifest_uri: URI.new!("file://#{tmp_dir}/stream.m3u8"), + manifest_uri: URI.new!("file://output/stream.m3u8"), min_segment_duration: Membrane.Time.seconds(5), target_segment_duration: Membrane.Time.seconds(10), storage: HLS.Storage.File.new() @@ -19,8 +19,8 @@ defmodule Membrane.HLS.SinkBinTest do location: "test/fixtures/samples_big-buck-bunny_bun33s.aac" }) |> child(:aac_parser, %Membrane.AAC.Parser{ - out_encapsulation: :none, - output_config: :esds + out_encapsulation: :ADTS + # output_config: :esds }) |> via_in(Pad.ref(:input, "audio_128k"), options: [