diff --git a/gst/videorate/gstvideorate.c b/gst/videorate/gstvideorate.c
index 2797d8ab47e61ad9bb973e80a0965a4784341200..4f53c813885d0f32892b868a13a7a93f069284cc 100644
--- a/gst/videorate/gstvideorate.c
+++ b/gst/videorate/gstvideorate.c
@@ -340,6 +340,12 @@ gst_video_rate_setcaps (GstPad * pad, GstCaps * caps)
 
       GST_DEBUG_OBJECT (videorate, "intersect %" GST_PTR_FORMAT, caps);
 
+      /* could turn up empty, due to e.g. colorspace etc */
+      if (gst_caps_get_size (caps) == 0) {
+        gst_caps_unref (caps);
+        goto no_transform;
+      }
+
       /* take first possibility */
       gst_caps_truncate (caps);
       structure = gst_caps_get_structure (caps, 0);