device: avoid trying to remove source multiple times

   ModemManager[115879]: <debug> [1613341789.760031] [modem3] completely disposed

   (ModemManager:115879): GLib-CRITICAL **: 23:29:49.760: Source ID 2379 was not found when attempting to remove it

   Thread 1 "ModemManager" received signal SIGTRAP, Trace/breakpoint trap.
   0x00007ffff774b343 in g_logv () from /usr/lib/libglib-2.0.so.0
   (gdb)
   (gdb) bt
   #0  0x00007ffff774b343 in g_logv () at /usr/lib/libglib-2.0.so.0
   #1  0x00007ffff774b5c0 in g_log () at /usr/lib/libglib-2.0.so.0
   #2  0x00007ffff7741c9e in g_source_remove () at /usr/lib/libglib-2.0.so.0
   #3  0x00005555555aad02 in dispose (object=0x555555831260) at mm-device.c:802
   #4  0x00007ffff7843755 in g_object_unref () at /usr/lib/libgobject-2.0.so.0
   #5  0x00005555555a5107 in glib_autoptr_clear_MMDevice (_ptr=0x555555831260) at mm-device.h:63
   #6  0x00005555555a5125 in glib_autoptr_cleanup_MMDevice (_ptr=0x7fffffffe090) at mm-device.h:63
   #7  0x00005555555a59ab in device_removed (self=0x555555769220, subsystem=0x55555577dc50 "tty", name=0x555555869a40 "ttyUSB3") at mm-base-manager.c:237
   #8  0x00005555555a620d in handle_uevent (self=0x555555769220, action=0x5555558987b0 "remove", device=0x555555893840) at mm-base-manager.c:445
   #9  0x00007ffff7381acd in  () at /usr/lib/libffi.so.7
   #10 0x00007ffff738103a in  () at /usr/lib/libffi.so.7
   #11 0x00007ffff783c8fe in g_cclosure_marshal_generic () at /usr/lib/libgobject-2.0.so.0
   #12 0x00007ffff7837072 in g_closure_invoke () at /usr/lib/libgobject-2.0.so.0
   #13 0x00007ffff785fa85 in  () at /usr/lib/libgobject-2.0.so.0
   #14 0x00007ffff78535dd in g_signal_emit_valist () at /usr/lib/libgobject-2.0.so.0
   #15 0x00007ffff7853b40 in g_signal_emit () at /usr/lib/libgobject-2.0.so.0
   #16 0x00007ffff7e792aa in  () at /usr/lib/libgudev-1.0.so.0
   #17 0x00007ffff7742b84 in g_main_context_dispatch () at /usr/lib/libglib-2.0.so.0
   #18 0x00007ffff7796c21 in  () at /usr/lib/libglib-2.0.so.0
   #19 0x00007ffff77420d3 in g_main_loop_run () at /usr/lib/libglib-2.0.so.0
   #20 0x00005555555a1df4 in main (argc=2, argv=0x7fffffffea88) at main.c:213