diff --git a/utils/fallbackswitch/src/fallbacksrc.rs b/utils/fallbackswitch/src/fallbacksrc.rs
index 2f1aa3d8558d1db5ed8ceecfa34cc049b81adddd..4bd103bfce39f60b49df4d59a6234791e20d8e76 100644
--- a/utils/fallbackswitch/src/fallbacksrc.rs
+++ b/utils/fallbackswitch/src/fallbacksrc.rs
@@ -1109,6 +1109,18 @@ impl FallbackSrc {
         state.video_stream = None;
         state.audio_stream = None;
 
+        if let Source::Element(ref source) = state.configured_source {
+            // Explicitly remove the source element from the CustomSource so that we can
+            // later create a new CustomSource and add it again there.
+            if source.has_as_parent(&state.source) {
+                let _ = source.set_state(gst::State::Null);
+                let _ = state
+                    .source
+                    .downcast_ref::<gst::Bin>()
+                    .unwrap()
+                    .remove(source);
+            }
+        }
         element.remove(&state.source).unwrap();
 
         if let Some(timeout) = state.source_pending_restart_timeout.take() {