diff --git a/gst/videorate/gstvideorate.c b/gst/videorate/gstvideorate.c
index a80453fc2372545bc91a563267851dd32541b5c5..ea99e00ee40539720e4ec8b50469111dea72839c 100644
--- a/gst/videorate/gstvideorate.c
+++ b/gst/videorate/gstvideorate.c
@@ -1469,6 +1469,14 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
 
   if (videorate->to_rate_numerator == 0 && videorate->prevbuf &&
       !videorate->force_variable_rate) {
+    if (!GST_BUFFER_PTS_IS_VALID (buffer) ||
+        !GST_BUFFER_PTS_IS_VALID (videorate->prevbuf)) {
+      GST_ELEMENT_ERROR (videorate, STREAM, FAILED, (NULL),
+          ("videorate requires a non-variable framerate on the output caps or the"
+              " two first consecutive buffers to have valid timestamps to guess the"
+              " framerate."));
+      return GST_FLOW_ERROR;
+    }
     gst_video_rate_check_variable_rate (videorate, buffer);
   }
 
diff --git a/tests/check/elements/videorate.c b/tests/check/elements/videorate.c
index 3d2ef460c4488f677dc9ba7ddd040faef8f44c23..19c15f960346b6dde04bc4f8c2534bfc47a271d7 100644
--- a/tests/check/elements/videorate.c
+++ b/tests/check/elements/videorate.c
@@ -1540,7 +1540,7 @@ GST_END_TEST;
 GST_START_TEST (test_nopts_in_middle)
 {
   GstElement *videorate;
-  GstBuffer *first, *second, *third;
+  GstBuffer *first, *second;
   GstCaps *caps;
 
   videorate =
@@ -1576,33 +1576,10 @@ GST_START_TEST (test_nopts_in_middle)
   gst_buffer_ref (second);
 
   /* pushing gives away my reference ... */
-  fail_unless (gst_pad_push (mysrcpad, second) == GST_FLOW_OK);
-  /* ... and the first one should have been pushed out */
+  fail_unless (gst_pad_push (mysrcpad, second) == GST_FLOW_ERROR);
   ASSERT_BUFFER_REFCOUNT (second, "second", 1);
-  fail_unless_equals_int (g_list_length (buffers), 1);
-
-  /* ... and the first one was replaced */
-  assert_videorate_stats (videorate, "second", 2, 1, 0, 0);
-  ASSERT_BUFFER_REFCOUNT (first, "first", 1);
-
-  /* third buffer */
-  third = gst_buffer_new_and_alloc (4);
-  GST_BUFFER_TIMESTAMP (third) = 2 * GST_SECOND;
-  gst_buffer_memset (third, 0, 0, 4);
-  ASSERT_BUFFER_REFCOUNT (third, "third", 1);
-  gst_buffer_ref (third);
-
-  /* pushing gives away my reference ... */
-  fail_unless (gst_pad_push (mysrcpad, third) == GST_FLOW_OK);
-  /* ... and a copy is now stuck inside videorate */
-  ASSERT_BUFFER_REFCOUNT (third, "third", 1);
+  fail_unless_equals_int (g_list_length (buffers), 0);
 
-  /* and now it should have pushed out the second one */
-  fail_unless_equals_int (g_list_length (buffers), 1);
-  ASSERT_BUFFER_REFCOUNT (first, "first", 1);
-  ASSERT_BUFFER_REFCOUNT (second, "second", 1);
-  ASSERT_BUFFER_REFCOUNT (third, "third", 1);
-  assert_videorate_stats (videorate, "third", 3, 2, 0, 0);
 
   /* cleanup */
   gst_buffer_unref (first);