sdpdemux: deadlock while stopping
Here are the relevant thread traces:
THREAD A
[Switching to thread 1 (Thread 0x7fffeb707940 (LWP 89960))]
(gdb) bt
#0 __lll_lock_wait (futex=futex@entry=0x7fff341df870, private=0) at lowlevellock.c:52
#1 0x00007fffeeb7d131 in __GI___pthread_mutex_lock (mutex=0x7fff341df870) at ../nptl/pthread_mutex_lock.c:115
#2 0x00007ffff7d9b18b in post_activate (pad=0x555556e563e0, new_mode=GST_PAD_MODE_NONE) at ../gst/gstpad.c:1046
#3 0x00007ffff7d9b9f0 in activate_mode_internal (pad=0x555556e563e0, parent=0x7fff7801f5a0, mode=GST_PAD_MODE_PUSH, active=0) at ../gst/gstpad.c:1224
#4 0x00007ffff7d9b55e in gst_pad_set_active (pad=0x555556e563e0, active=0) at ../gst/gstpad.c:1115
#5 0x00007ffff7d7a487 in activate_pads (vpad=0x7fffffffbd90, ret=0x7fffffffbdf0, active=0x7fffffffbe24) at ../gst/gstelement.c:3040
#6 0x00007ffff7d8e988 in gst_iterator_fold (it=0x7fff58002a10, func=0x7ffff7d7a444 <activate_pads>, ret=0x7fffffffbdf0, user_data=0x7fffffffbe24) at ../gst/gstiterator.c:617
#7 0x00007ffff7d7a52e in iterator_activate_fold_with_resync (iter=0x7fff58002a10, func=0x7ffff7d7a444 <activate_pads>, user_data=0x7fffffffbe24) at ../gst/gstelement.c:3064
#8 0x00007ffff7d7a674 in gst_element_pads_activate (element=0x7fff7801f5a0, active=0) at ../gst/gstelement.c:3109
#9 0x00007ffff7d7a976 in gst_element_change_state_func (element=0x7fff7801f5a0, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/gstelement.c:3174
#10 0x00007fffd450be40 in gst_rtp_pt_demux_change_state (element=0x7fff7801f5a0, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/rtpmanager/gstrtpptdemux.c:751
#11 0x00007ffff7d79f7c in gst_element_change_state (element=0x7fff7801f5a0, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/gstelement.c:2952
#12 0x00007ffff7d79d06 in gst_element_set_state_func (element=0x7fff7801f5a0, state=GST_STATE_READY) at ../gst/gstelement.c:2906
#13 0x00007ffff7d798fe in gst_element_set_state (element=0x7fff7801f5a0, state=GST_STATE_READY) at ../gst/gstelement.c:2807
#14 0x00007ffff7d474a9 in gst_bin_element_set_state (bin=0x7fff7000b650, element=0x7fff7801f5a0, base_time=1591692852883416818, start_time=0, current=GST_STATE_PAUSED, next=GST_STATE_READY) at ../gst/gstbin.c:2605
#15 0x00007ffff7d48b7d in gst_bin_change_state_func (element=0x7fff7000b650, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/gstbin.c:2947
#16 0x00007fffd44ec2d7 in gst_rtp_bin_change_state (element=0x7fff7000b650, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/rtpmanager/gstrtpbin.c:3228
#17 0x00007ffff7d79f7c in gst_element_change_state (element=0x7fff7000b650, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/gstelement.c:2952
#18 0x00007ffff7d79d06 in gst_element_set_state_func (element=0x7fff7000b650, state=GST_STATE_READY) at ../gst/gstelement.c:2906
#19 0x00007ffff7d798fe in gst_element_set_state (element=0x7fff7000b650, state=GST_STATE_READY) at ../gst/gstelement.c:2807
#20 0x00007ffff7d474a9 in gst_bin_element_set_state (bin=0x5555565522a0, element=0x7fff7000b650, base_time=1591692852883416818, start_time=0, current=GST_STATE_PAUSED, next=GST_STATE_READY) at ../gst/gstbin.c:2605
#21 0x00007ffff7d48b7d in gst_bin_change_state_func (element=0x5555565522a0, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/gstbin.c:2947
#22 0x00007fffd45a260f in gst_sdp_demux_change_state (element=0x5555565522a0, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/sdp/gstsdpdemux.c:1387
#23 0x00007ffff7d79f7c in gst_element_change_state (element=0x5555565522a0, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/gstelement.c:2952
#24 0x00007ffff7d794a7 in gst_element_continue_state (element=0x5555565522a0, ret=GST_STATE_CHANGE_NO_PREROLL) at ../gst/gstelement.c:2660
#25 0x00007ffff7d7a311 in gst_element_change_state (element=0x5555565522a0, transition=GST_STATE_CHANGE_PLAYING_TO_PAUSED) at ../gst/gstelement.c:2998
#26 0x00007ffff7d79d06 in gst_element_set_state_func (element=0x5555565522a0, state=GST_STATE_READY) at ../gst/gstelement.c:2906
#27 0x00007ffff7d798fe in gst_element_set_state (element=0x5555565522a0, state=GST_STATE_READY) at ../gst/gstelement.c:2807
#28 0x00007ffff7d474a9 in gst_bin_element_set_state (bin=0x5555567d6700, element=0x5555565522a0, base_time=1591692852883416818, start_time=0, current=GST_STATE_PAUSED, next=GST_STATE_READY) at ../gst/gstbin.c:2605
#29 0x00007ffff7d48b7d in gst_bin_change_state_func (element=0x5555567d6700, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/gstbin.c:2947
#30 0x00007ffff7d79f7c in gst_element_change_state (element=0x5555567d6700, transition=GST_STATE_CHANGE_PAUSED_TO_READY) at ../gst/gstelement.c:2952
#31 0x00007ffff7d79d06 in gst_element_set_state_func (element=0x5555567d6700, state=GST_STATE_NULL) at ../gst/gstelement.c:2906
#32 0x00007ffff7d798fe in gst_element_set_state (element=0x5555567d6700, state=GST_STATE_NULL) at ../gst/gstelement.c:2807
...
#50 0x00007ffff7bc5533 in g_main_loop_run () at /usr/lib/x86_64-linux-gnu/libglib-2.0.so.0
...
THREAD B
[Switching to thread 404 (Thread 0x7fff7debe700 (LWP 94899))]
(gdb) bt
#0 __lll_lock_wait (futex=futex@entry=0x5555564c2b10, private=0) at lowlevellock.c:52
#1 0x00007fffeeb7d131 in __GI___pthread_mutex_lock (mutex=0x5555564c2b10) at ../nptl/pthread_mutex_lock.c:115
#2 0x00007fffd459f894 in new_session_pad (session=0x7fff7000b650, pad=0x7fff1c00e610, demux=0x5555565522a0) at ../gst/sdp/gstsdpdemux.c:572
#3 0x00007fffee242ff5 in () at /usr/lib/x86_64-linux-gnu/libffi.so.7
#4 0x00007fffee24240a in () at /usr/lib/x86_64-linux-gnu/libffi.so.7
#5 0x00007ffff7cb130d in g_cclosure_marshal_generic () at /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0
#6 0x00007ffff7cb0802 in g_closure_invoke () at /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0
#7 0x00007ffff7cc4814 in () at /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0
#8 0x00007ffff7ccfb9e in g_signal_emit_valist () at /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0
#9 0x00007ffff7cd00d3 in g_signal_emit () at /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0
#10 0x00007ffff7d7438a in gst_element_add_pad (element=0x7fff7000b650, pad=0x7fff1c00e610) at ../gst/gstelement.c:711
#11 0x00007fffd44ec733 in expose_recv_src_pad (rtpbin=0x7fff7000b650, pad=0x7fff2802c590, stream=0x7fff3403d000, pt=99 'c') at ../gst/rtpmanager/gstrtpbin.c:3334
#12 0x00007fffd44ec8dc in new_payload_found (element=0x7fff7801f5a0, pt=99, pad=0x7fff2802c590, stream=0x7fff3403d000) at ../gst/rtpmanager/gstrtpbin.c:3378
#13 0x00007fffee242ff5 in () at /usr/lib/x86_64-linux-gnu/libffi.so.7
#14 0x00007fffee24240a in () at /usr/lib/x86_64-linux-gnu/libffi.so.7
#15 0x00007ffff7cb130d in g_cclosure_marshal_generic () at /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0
#16 0x00007ffff7cb0802 in g_closure_invoke () at /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0
#17 0x00007ffff7cc4814 in () at /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0
#18 0x00007ffff7ccfb9e in g_signal_emit_valist () at /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0
#19 0x00007ffff7cd00d3 in g_signal_emit () at /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0
#20 0x00007fffd450b447 in gst_rtp_pt_demux_chain (pad=0x555556e563e0, parent=0x7fff7801f5a0, buf=0x7fff34350000) at ../gst/rtpmanager/gstrtpptdemux.c:525
#21 0x00007ffff7da6348 in gst_pad_chain_data_unchecked (pad=0x555556e563e0, type=4112, data=0x7fff34350000) at ../gst/gstpad.c:4327
#22 0x00007ffff7da71cb in gst_pad_push_data (pad=0x7fff88378a00, type=4112, data=0x7fff34350000) at ../gst/gstpad.c:4583
#23 0x00007ffff7da7974 in gst_pad_push (pad=0x7fff88378a00, buffer=0x7fff34350000) at ../gst/gstpad.c:4702
#24 0x00007fffd44fddc9 in pop_and_push_next (jitterbuffer=0x7fff340517f0, seqnum=12943) at ../gst/rtpmanager/gstrtpjitterbuffer.c:3568
#25 0x00007fffd44fe3e5 in handle_next_buffer (jitterbuffer=0x7fff340517f0) at ../gst/rtpmanager/gstrtpjitterbuffer.c:3667
#26 0x00007fffd4501348 in gst_rtp_jitter_buffer_loop (jitterbuffer=0x7fff340517f0) at ../gst/rtpmanager/gstrtpjitterbuffer.c:4226
#27 0x00007ffff7de5e01 in gst_task_func (task=0x7fff80399830) at ../gst/gsttask.c:328
#28 0x00007ffff7de712b in default_func (tdata=0x7fffe4008390, pool=0x555556188d90) at ../gst/gsttaskpool.c:70
#29 0x00007ffff7bef1d4 in () at /usr/lib/x86_64-linux-gnu/libglib-2.0.so.0
#30 0x00007ffff7bee931 in () at /usr/lib/x86_64-linux-gnu/libglib-2.0.so.0
#31 0x00007fffeeb7a609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#32 0x00007fffee85d103 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
- Thread A (gstpad.c:1046, frame #2 (closed)) waits for PAD_STREAM_LOCK of rtpptdemux, held by thread B (gstpad.c:4276, frame #21)
- Thread B (gstsdpdemux.c:572, frame #2 (closed)) waits for SDP_STREAM_LOCK, held by thread A (gstdspdemux.c:1369, frame #22 (closed))
My current workaround is to set a going_down
flag in sdpdemux for GST_STATE_CHANGE_PAUSED_TO_READY and GST_STATE_CHANGE_READY_TO_NULL before calling change_state()
of the parent and to immediately return from new_session_pad()
if the flag is set. However, I am not sure if that prevents deadlocking under all circumstances.