rtspsrc: deadlock on pipeline set_state(NULL)
I have a gstreamer pipeline streaming using rtspsrc in a network that has connection issues. rtspsrc often encounters GstRTSPSrcTimeout
errors. When this timeout is received, I set the pipeline state to NULL which in turn calls set_state(NULL) on the rtspsrc. This operation deadlocks and never exits.
Observing the thread dump it seems rtspsrc is deadlocking on changing state GST_STATE_CHANGE_PLAYING_TO_PAUSED -> GST_RTSP_STREAM_LOCK (rtspsrc):
if (rtspsrc->is_live) {
/* unblock the tcp tasks and make the loop waiting */
if (gst_rtspsrc_loop_send_cmd (rtspsrc, CMD_WAIT, CMD_LOOP)) {
/* make sure it is waiting before we send PAUSE or PLAY below */
GST_RTSP_STREAM_LOCK (rtspsrc);
GST_RTSP_STREAM_UNLOCK (rtspsrc);
}
}
break;
Thread 107 (Thread 0x7f3ddc1d1700 (LWP 19886)):
#0 0x00007f3f7089f3ad in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f3f70898d6a in pthread_mutex_lock () from /lib64/libpthread.so.0
#2 0x00007f3dddc0a5aa in g_rec_mutex_lock (mutex=mutex@entry=0x7f3c54004468) at ../glib/gthread-posix.c:395
#3 0x00007f3ddc41394f in gst_rtspsrc_change_state (element=0x7f3c540042e0, transition=GST_STATE_CHANGE_PLAYING_TO_PAUSED) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:9499
#4 0x00007f3dde18ef9e in gst_element_change_state (element=element@entry=0x7f3c540042e0, transition=transition@entry=GST_STATE_CHANGE_PLAYING_TO_PAUSED) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstelement.c:3093
#5 0x00007f3dde18f68b in gst_element_set_state_func (element=0x7f3c540042e0, state=GST_STATE_PAUSED) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstelement.c:3047
#6 0x00007f3dde16c555 in gst_bin_element_set_state (next=GST_STATE_PAUSED, current=GST_STATE_PLAYING, start_time=4748879646055, base_time=829858899909097, element=0x7f3c540042e0, bin=0x7f3cbc002500) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstbin.c:2581
#7 gst_bin_change_state_func (element=0x7f3cbc002500, transition=GST_STATE_CHANGE_PLAYING_TO_PAUSED) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstbin.c:2930
#8 0x00007f3dde1ba18d in gst_pipeline_change_state (element=0x7f3cbc002500, transition=GST_STATE_CHANGE_PLAYING_TO_PAUSED) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpipeline.c:529
#9 0x00007f3dde18ef9e in gst_element_change_state (element=element@entry=0x7f3cbc002500, transition=transition@entry=GST_STATE_CHANGE_PLAYING_TO_PAUSED) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstelement.c:3093
#10 0x00007f3dde18f68b in gst_element_set_state_func (element=0x7f3cbc002500, state=GST_STATE_NULL) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstelement.c:3047
#11 0x00007f3dde48e052 in ?? ()
#12 0x0000000000000006 in ?? ()
#13 0x00007f3ddc1d0228 in ?? ()
#14 0x00007f3ddc1cfbf0 in ?? ()
#15 0x00007f3dde48d14c in ?? ()
#16 0x00007f3ddc1d0380 in ?? ()
#17 0x00007f3f50be92a4 in ?? ()
#18 0x00007f3d00000000 in ?? ()
#19 0x00007f3ddc1cfa20 in ?? ()
#20 0x00007f3ddc1cfcf0 in ?? ()
#21 0x0000000000000000 in ?? ()
Thread 166 (Thread 0x7f3c50bfc700 (LWP 21156)):
#0 0x00007f3f703c82a9 in syscall () from /lib64/libc.so.6
---Type <return> to continue, or q <return> to quit---
#1 0x00007f3dddc0b26f in g_cond_wait (cond=cond@entry=0x7f3c4c01e5a0, mutex=mutex@entry=0x7f3c4c01e590) at ../glib/gthread-posix.c:1574
#2 0x00007f3d31462468 in gst_rtp_jitter_buffer_chain (pad=<optimized out>, parent=0x7f3c4c01e8f0, buffer=<optimized out>) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c:3427
#3 0x00007f3dde1aa618 in gst_pad_chain_data_unchecked (pad=pad@entry=0x7f3c4c0199c0, type=type@entry=4112, data=data@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4463
#4 0x00007f3dde1ae184 in gst_pad_push_data (pad=pad@entry=0x7f3c4c0192d0, type=type@entry=4112, data=data@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4739
#5 0x00007f3dde1b4f0e in gst_pad_push (pad=pad@entry=0x7f3c4c0192d0, buffer=buffer@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4858
#6 0x00007f3d31473b4d in gst_rtp_ssrc_demux_chain (pad=<optimized out>, parent=0x7f3c4c014090, buf=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c:680
#7 0x00007f3dde1aa618 in gst_pad_chain_data_unchecked (pad=pad@entry=0x7f3c3c00d610, type=type@entry=4112, data=data@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4463
#8 0x00007f3dde1ae184 in gst_pad_push_data (pad=pad@entry=0x7f3c3c00dab0, type=type@entry=4112, data=data@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4739
#9 0x00007f3dde1b4f0e in gst_pad_push (pad=0x7f3c3c00dab0, buffer=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4858
#10 0x00007f3dde1aa618 in gst_pad_chain_data_unchecked (pad=pad@entry=0x7f3c3c00dd00, type=type@entry=4112, data=data@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4463
#11 0x00007f3dde1ae184 in gst_pad_push_data (pad=pad@entry=0x7f3c4c0182a0, type=type@entry=4112, data=data@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4739
#12 0x00007f3dde1b4f0e in gst_pad_push (pad=pad@entry=0x7f3c4c0182a0, buffer=buffer@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4858
#13 0x00007f3d31494451 in gst_rtp_session_process_rtp (sess=<optimized out>, src=<optimized out>, buffer=0x7f3c4d6b1000, user_data=0x7f3cb0011440) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c:1420
#14 0x00007f3d31479722 in source_push_rtp (source=0x7f3ce856fcf0, data=0x7f3c4d6b1000, session=0x7f3cb0013be0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c:1566
#15 0x00007f3d3148b004 in push_packet (buffer=<optimized out>, src=0x7f3ce856fcf0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c:935
#16 rtp_source_process_rtp (src=src@entry=0x7f3ce856fcf0, pinfo=pinfo@entry=0x7f3c50bfb6f0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c:1344
#17 0x00007f3d314868f6 in rtp_session_process_rtp (sess=0x7f3cb0013be0, buffer=buffer@entry=0x7f3c4d6b1000, current_time=<optimized out>, running_time=<optimized out>, ntpnstime=<optimized out>) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c:2357
#18 0x00007f3d31498995 in gst_rtp_session_chain_recv_rtp (pad=<optimized out>, parent=0x7f3cb0011440, buffer=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c:2013
#19 0x00007f3dde1aa618 in gst_pad_chain_data_unchecked (pad=pad@entry=0x7f3c4c018050, type=type@entry=4112, data=data@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4463
#20 0x00007f3dde1ae184 in gst_pad_push_data (pad=pad@entry=0x7f3ce854b0f0, type=type@entry=4112, data=data@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4739
#21 0x00007f3dde1b4f0e in gst_pad_push (pad=pad@entry=0x7f3ce854b0f0, buffer=buffer@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4858
#22 0x00007f3dde196e02 in gst_proxy_pad_chain_default (pad=<optimized out>, parent=<optimized out>, buffer=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstghostpad.c:127
#23 0x00007f3dde1aa618 in gst_pad_chain_data_unchecked (pad=pad@entry=0x7f3ce8549170, type=type@entry=4112, data=data@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4463
#24 0x00007f3dde1ae184 in gst_pad_push_data (pad=pad@entry=0x7f3c4c018990, type=type@entry=4112, data=data@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4739
#25 0x00007f3dde1b4f0e in gst_pad_push (pad=pad@entry=0x7f3c4c018990, buffer=buffer@entry=0x7f3c4d6b1000) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4858
#26 0x00007f3ddc410c65 in gst_rtspsrc_handle_data (src=src@entry=0x7f3c540042e0, message=message@entry=0x7f3c50bfbd90) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:5789
#27 0x00007f3ddc42441c in gst_rtspsrc_loop_interleaved (src=0x7f3c540042e0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:5869
#28 gst_rtspsrc_loop (src=0x7f3c540042e0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:6366
#29 gst_rtspsrc_thread (src=0x7f3c540042e0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:9378
#30 0x00007f3dde1e6f96 in gst_task_func (task=0x7f3d28006710) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gsttask.c:384
#31 0x00007f3dddbdf22d in g_thread_pool_thread_proxy (data=<optimized out>) at ../glib/gthreadpool.c:354
#32 0x00007f3dddbdeb1d in g_thread_proxy (data=0x7f3c54018e40) at ../glib/gthread.c:826
#33 0x00007f3f7089644b in start_thread () from /lib64/libpthread.so.0
#34 0x00007f3f703cd52f in clone () from /lib64/libc.so.6
Thread 172 (Thread 0x7f3c48bfc700 (LWP 21164)):
#0 0x00007f3f7089f3ad in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f3f70898d6a in pthread_mutex_lock () from /lib64/libpthread.so.0
#2 0x00007f3dddc0a5aa in g_rec_mutex_lock (mutex=mutex@entry=0x7f3ce85491e0) at ../glib/gthread-posix.c:395
#3 0x00007f3dde1ab333 in gst_pad_send_event_unchecked (pad=pad@entry=0x7f3ce8549170, event=<optimized out>, event@entry=0x7f3c2c006c40, type=type@entry=GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:5892
#4 0x00007f3dde1abe13 in gst_pad_push_event_unchecked (pad=pad@entry=0x7f3c4c018990, event=0x7f3c2c006c40, type=type@entry=GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:5572
#5 0x00007f3dde1ac7f0 in push_sticky (user_data=<synthetic pointer>, ev=<synthetic pointer>, pad=0x7f3c4c018990) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4057
#6 events_foreach (user_data=<synthetic pointer>, func=0x7f3dde1ab780 <push_sticky>, pad=0x7f3c4c018990) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:613
#7 check_sticky (pad=0x7f3c4c018990, event=0x7f3c2c006c40) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:4116
#8 0x00007f3dde1b6108 in gst_pad_push_event (pad=0x7f3c4c018990, event=0x7f3c2c006c40) at ../../../../../../../../src/Gstreamer/subprojects/gstreamer/gst/gstpad.c:5705
#9 0x00007f3ddc413c5e in gst_rtspsrc_stream_push_event (src=0x7f3c540042e0, event=0x7f3c08008350, stream=0x7f3c4c00cf50) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:5217
#10 on_timeout_common (session=<optimized out>, source=<optimized out>, stream=<optimized out>) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c:3726
#11 0x00007f3ddded7a97 in g_closure_invoke (closure=0x7f3c4c01a650, return_value=0x0, n_param_values=2, param_values=0x7f3c48bfb9d0, invocation_hint=0x7f3c48bfb970) at ../gobject/gclosure.c:810
#12 0x00007f3dddee9930 in signal_emit_unlocked_R (node=node@entry=0x7f3d2811ef40, detail=detail@entry=0, instance=instance@entry=0x7f3cb0013be0, emission_return=emission_return@entry=0x0, instance_and_params=instance_and_params@entry=0x7f3c48bfb9d0) at ../gobject/gsignal.c:3741
#13 0x00007f3dddef3f31 in g_signal_emit_valist (instance=<optimized out>, signal_id=<optimized out>, detail=<optimized out>, var_args=var_args@entry=0x7f3c48bfbb78) at ../gobject/gsignal.c:3497
#14 0x00007f3dddef4982 in g_signal_emit (instance=instance@entry=0x7f3cb0013be0, signal_id=<optimized out>, detail=detail@entry=0) at ../gobject/gsignal.c:3553
#15 0x00007f3d3147ac24 in on_timeout (source=0x7f3ce856fcf0, sess=0x7f3cb0013be0) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c:1126
#16 session_cleanup (key=<optimized out>, source=0x7f3ce856fcf0, data=0x7f3c48bfbd40) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c:4243
#17 0x00007f3dddb992af in g_hash_table_foreach (hash_table=hash_table@entry=0x7f3c2c0062a0, func=func@entry=0x7f3d3147a3b0 <session_cleanup>, user_data=user_data@entry=0x7f3c48bfbd40) at ../glib/ghash.c:2065
#18 0x00007f3d314839bb in rtp_session_on_timeout (sess=sess@entry=0x7f3cb0013be0, current_time=current_time@entry=834607779252078, ntpnstime=<optimized out>, running_time=<optimized out>) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c:4696
#19 0x00007f3d31496195 in rtcp_thread (rtpsession=0x7f3cb0011440) at ../../../../../../../../src/Gstreamer/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c:1249
#20 0x00007f3dddbdeb1d in g_thread_proxy (data=0x7f3c4c006980) at ../glib/gthread.c:826
#21 0x00007f3f7089644b in start_thread () from /lib64/libpthread.so.0
#22 0x00007f3f703cd52f in clone () from /lib64/libc.so.6
I am using Gstreamer 1.22.
What can be done to address this issue or further investigate? It is difficult to produce intentionally but is frequently occurring in deployed use case.