Deadlock causes conference to go silent

[drottenberg]

Description
We had an audio conference (no video) with one moderator and many listeners. At some point no one was able to hear anything. When I attempted to restart freeswitch the error said that conference is still waiting for locks. The only way I was able to remove the locks was by running fsctl crash. After reviewing the core dump it seems like there was some kind of deadlock, see the attached backtrace.
It seems like thread 4 which is running the conference_thread_run is waiting for the audio_out_mutex for omember 0x7f3c8ae35910, while thread 162 which is running conference_member_del for member 0x7f3c8ae35910 is waiting on the conference mutex, which is most likely being held by thread 4 (otherwise it wouldn't have gotten to the point where it's attempting to acquire the audio_out_mutex).

I have attached the gdb output for the core dump

To Reproduce
I have not yet determined the exact steps to reproduce, but this happened twice in a row, for large conferences with many listeners joining and leaving constantly. The first time we observed this is after we recently upgraded to v1.10.13, we skipped several versions, so not sure with which version this issue started.

Expected behavior
Conferences should continue to produce audio regardless of how many listeners are joining and leaving

Package version or git hash
1.10.13

Trace logs
N/A

backtrace from core file
backtrace2.zip

#0  futex_wait (private=0, expected=2, futex_word=0x7f3b302afae0) at ../sysdeps/nptl/futex-internal.h:146
        __ret = -512
        err = <optimized out>
        err = <optimized out>
        __ret = <optimized out>
        resultvar = <optimized out>
        __arg4 = <optimized out>
        __arg3 = <optimized out>
        __arg2 = <optimized out>
        __arg1 = <optimized out>
        _a4 = <optimized out>
        _a3 = <optimized out>
        _a2 = <optimized out>
        _a1 = <optimized out>
#1  __GI___lll_lock_wait (futex=futex@entry=0x7f3b302afae0, private=0) at ./nptl/lowlevellock.c:49
No locals.
#2  0x00007f3d374ab4aa in lll_mutex_lock_optimized (mutex=0x7f3b302afae0) at ./nptl/pthread_mutex_lock.c:48
        __futex = 0x7f3b302afae0
        private = <optimized out>
        private = <optimized out>
        __futex = <optimized out>
#3  ___pthread_mutex_lock (mutex=0x7f3b302afae0) at ./nptl/pthread_mutex_lock.c:128
        id = <optimized out>
        type = <optimized out>
        __PRETTY_FUNCTION__ = "___pthread_mutex_lock"
        id = <optimized out>
#4  0x00007f3d1ecc0f30 in conference_member_add (conference=conference@entry=0x7f3b302af1a8, member=member@entry=0x7f3c8afa3910) at conference_member.c:732
        status = SWITCH_STATUS_FALSE
        event = 0x7f3c8afa3280
        msg = "\002\000\000\000\000\000\000\000P\231\244\2266\177\000\000\000\000\000\000\000\000\000\000Ѓ4d:\177\000\000\025\261\2367=\177\000\000(", '\000' <repeats 199 times>...
        call_list = 0x0
        channel = <optimized out>
        controls = 0x0
        position = 0x0
        var = 0x0
        has_video = SWITCH_TRUE
        __PRETTY_FUNCTION__ = "conference_member_add"
        __func__ = "conference_member_add"
#5  0x00007f3d1ec9848a in conference_function (session=0x7f3696a4b878, data=<optimized out>) at mod_conference.c:2456
        member = {id = 51319, session = 0x7f3696a4b878, channel = 0x7f3696a49950, conference = 0x0, pool = 0x7f3696a49868, audio_buffer = 0x7f3a641e52d0, mux_buffer = 0x7f3a64343aa0, resample_buffer = 0x0, flags = {MFLAG_CAN_SPEAK, MFLAG_RUNNING, MFLAG_CAN_SPEAK, MFLAG_RUNNING <repeats 30 times>, MFLAG_CAN_SPEAK, MFLAG_CAN_SPEAK, MFLAG_RUNNING <repeats 13 times>}, score = 0, last_score = 0, score_iir = 0, flag_mutex = 0x7f3a64358f00, write_mutex = 0x7f3a64358f50, audio_in_mutex = 0x7f3a64359040, audio_out_mutex = 0x7f3a64359090, read_mutex = 0x7f3a64358fa0, fnode_mutex = 0x7f3a64358ff0, rwlock = 0x7f3a64359130, read_impl = {codec_type = SWITCH_CODEC_TYPE_AUDIO, ianacode = 0 '\000', iananame = 0x0, fmtp = 0x0, samples_per_second = 0, actual_samples_per_second = 0, bits_per_second = 0, microseconds_per_packet = 0, samples_per_packet = 0, decoded_bytes_per_packet = 0, encoded_bytes_per_packet = 0, number_of_channels = 0 '\000', codec_frames_per_packet = 0, init = 0x0, encode = 0x0, decode = 0x0, encode_video = 0x0, decode_video = 0x0, codec_control = 0x0, destroy = 0x0, codec_id = 0, impl_id = 0, modname = 0x0, next = 0x0}, orig_read_impl = {codec_type = SWITCH_CODEC_TYPE_AUDIO, ianacode = 0 '\000', iananame = 0x55c587558f08 "PCMU", fmtp = 0x0, samples_per_second = 8000, actual_samples_per_second = 8000, bits_per_second = 64000, microseconds_per_packet = 20000, samples_per_packet = 160, decoded_bytes_per_packet = 320, encoded_bytes_per_packet = 160, number_of_channels = 1 '\001', codec_frames_per_packet = 160, init = 0x7f3d3777ca20, encode = 0x7f3d3777c3c0, decode = 0x7f3d3777ca40, encode_video = 0x0, decode_video = 0x0, codec_control = 0x0, destroy = 0x7f3d3777ca10, codec_id = 123, impl_id = 138, modname = 0x55c587558600 "CORE_PCM_MODULE", next = 0x55c587558de0}, read_codec = {codec_interface = 0x55c587553da8, implementation = 0x55c587554ac0, fmtp_in = 0x0, fmtp_out = 0x0, flags = 259, memory_pool = 0x7f3696a49868, private_info = 0x0, agreed_pt = 0 '\000', mutex = 0x7f3a64359190, next = 0x7f36f3fe7fa0, session = 0x7f3696a4b878, cur_frame = 0x0}, write_codec = {codec_interface = 0x55c587553da8, implementation = 0x55c587554ac0, fmtp_in = 0x0, fmtp_out = 0x0, flags = 259, memory_pool = 0x7f3696a49868, private_info = 0x0, agreed_pt = 0 '\000', mutex = 0x7f3a6455cf28, next = 0x0, session = 0x7f3696a4b878, cur_frame = 0x0}, rec_path = 0x0, rec_time = 0, rec = 0x0, frame = 0x7f3a64557f18 "", last_frame = 0x0, frame_size = 8192, mux_frame = 0x7f3a6455af28 "", read = 0, vol_period = 0, energy_level = 0, auto_energy_level = 0, max_energy_level = 0, agc_level = 0, agc_low_energy_level = 0, agc_margin = 0, agc_change_factor = 0, agc_period_len = 0, agc = 0x0, mute_counter = 0, burst_mute_count = 0, score_avg = 0, max_energy_hits = 0, max_energy_hit_trigger = 0, volume_in_level = 0, volume_out_level = 0, join_time = 0, last_talking = 0, first_talk_detect = 0, talk_detects = 0, auto_energy_track = 0, talk_track = 0, score_count = 0, score_accum = 0, score_delta_accum = 0, native_rate = 8000, gate_open = 0, gate_count = 0, nogate_count = 0, talking_count = 0, read_resampler = 0x0, resample_out = 0x0, resample_out_len = 0, fnode = 0x0, relationships = 0x0, lsh = {speech_interface = 0x0, flags = 0, name = 0x0, rate = 0, speed = 0, samples = 0, channels = 0, real_channels = 0, voice = '\000' <repeats 79 times>, engine = 0x0, param = 0x0, memory_pool = 0x0, resampler = 0x0, buffer = 0x0, dbuf = 0x0, dbuflen = 0, samplerate = 0, native_rate = 0, private_info = 0x0}, sh = 0x0, verbose_events = 0, next = 0x0, dmachine = 0x0, cdr_node = 0x0, kicked_sound = 0x0, dtmf_queue = 0x0, video_queue = 0x0, video_muxing_write_thread = 0x0, video_layer_thread = 0x0, layer_thread_running = 0, input_thread = 0x0, layer_cond = 0x0, layer_cond_mutex = 0x0, json = 0x0, status_field = 0x0, loop_loop = 0 '\000', al = 0x0, last_speech_channels = 0, video_layer_id = 0, canvas_id = 0, watching_canvas_id = 0, layer_timeout = 0, video_codec_index = 0, video_codec_id = 0, video_banner_text = 0x0, video_logo = 0x0, logo_pos = POS_LEFT_TOP, logo_fit = SWITCH_FIT_SIZE, video_mute_png = 0x0, video_reservation_id = 0x0, video_role_id = 0x0, video_codec_group = 0x0, vid_params = {width = 0, height = 0, fps = 0, d_width = 0, d_height = 0}, auto_kps_debounce_ticks = 0, layer_loops = 0, fb = 0x0, avatar_png_img = 0x0, video_mute_img = 0x0, floor_packets = 0, blanks = 0, managed_kps = 0, managed_kps_set = 0, blackouts = 0, good_img = 0, auto_avatar = 0, avatar_patched = 0, video_media_flow = SWITCH_MEDIA_FLOW_SENDRECV, canvas = 0x0, pcanvas_img = 0x0, max_bw_in = 0, force_bw_in = 0, max_bw_out = 0, reset_media = 0, flip = 0, flip_count = 0, text_mutex = 0x7f3a643590e0, text_buffer = 0x0, text_framedata = 0x0, text_framesize = 0, cam_opts = {manual_pan = 0, manual_zoom = 0, autozoom = 0, autopan = 0, zoom_factor = 0, snap_factor = 0, zoom_move_factor = 0, pan_speed = 0, pan_accel_speed = 0, pan_accel_min = 0, zoom_speed = 0, zoom_accel_speed = 0, zoom_accel_min = 0}, video_filters = 0, video_manual_border = 0}
        conference = 0x7f3b302af1a8
        channel = <optimized out>
        mydata = <optimized out>
        conference_name = <optimized out>
        bridge_prefix = 0x7f3d1ecc7d51 "bridge:"
        flags_prefix = 0x7f3d1ecc7449 "+flags{"
        bridgeto = 0x0
        profile_name = <optimized out>
        cxml = 0x0
        cfg = 0x7f3a6449b280
        profiles = <optimized out>
        flags_str = <optimized out>
        v_flags_str = <optimized out>
        cflags_str = <optimized out>
        v_cflags_str = <optimized out>
        mflags = {MFLAG_CAN_SPEAK, MFLAG_RUNNING, MFLAG_CAN_SPEAK, MFLAG_RUNNING <repeats 30 times>, MFLAG_CAN_SPEAK, MFLAG_CAN_SPEAK, MFLAG_RUNNING <repeats 13 times>}
        msg = {from = 0x7f3d1ecc7011 "mod_conference.c", message_id = SWITCH_MESSAGE_INDICATE_SESSION_ID, numeric_arg = 0, string_arg = 0x0, string_arg_size = 0, pointer_arg = 0x0, pointer_arg_size = 0, numeric_reply = 0, string_reply = 0x0, string_reply_size = 0, pointer_reply = 0x0, pointer_reply_size = 0, flags = 0, _file = 0x0, _func = 0x0, _line = 0, string_array_arg = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, delivery_time = 0}
        isbr = <optimized out>
        dpin = <optimized out>
        mdpin = <optimized out>
        xml_cfg = <optimized out>
        params = 0x7f3a64326d60
        locked = 0
        mpin_matched = 0
        mid = 0x7f3a6455cf78
        __func__ = "conference_function"
        done = <optimized out>
        __PRETTY_FUNCTION__ = "conference_function"
#6  0x00007f3d3769356e in switch_core_session_exec () from /usr/lib/libfreeswitch.so.1
No symbol table info available.
#7  0x00007f3d37693d91 in switch_core_session_execute_application_get_flags () from /usr/lib/libfreeswitch.so.1
No symbol table info available.
#8  0x00007f3d3775e184 in switch_ivr_parse_event () from /usr/lib/libfreeswitch.so.1
No symbol table info available.
#9  0x00007f3d3775edab in switch_ivr_parse_next_event () from /usr/lib/libfreeswitch.so.1
No symbol table info available.
#10 0x0000000000000000 in ?? ()