videoscale: fix 4-tap scaling for 64-bpp formats

Fix invalid memory access caused by broken pointer arithmetic.

If we have a uint16_t *tmpbuf and add n * dest->stride to it, we
skip twice as much as we intended to because dest->stride is in
bytes and not in pixels. This made us write beyond the end of
our allocated temp buffer, and made the unit test crash.

gst/videoscale/gstvideoscale.c

@@ -532,7 +532,7 @@ gst_video_scale_set_info (GstVideoFilter * filter, GstCaps * in,
 
   if (videoscale->tmp_buf)
     g_free (videoscale->tmp_buf);
-  videoscale->tmp_buf = g_malloc (out_info->width * 8 * 4);
+  videoscale->tmp_buf = g_malloc (out_info->width * sizeof (guint64) * 4);
 
   if (in_info->width == out_info->width && in_info->height == out_info->height) {
     gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (filter), TRUE);

gst/videoscale/vs_4tap.c

@@ -1449,6 +1449,8 @@ vs_image_scale_4tap_AYUV64 (const VSImage * dest, const VSImage * src,
   int xacc;
   int k;
   guint16 *tmpbuf = (guint16 *) tmpbuf8;
+  /* destination stride in pixels for easier use with tmpbuf variable */
+  int dest_pixstride = dest->stride / sizeof (guint16);
 
   if (dest->height == 1)
     y_increment = 0;
@@ -1463,7 +1465,7 @@ vs_image_scale_4tap_AYUV64 (const VSImage * dest, const VSImage * src,
   k = 0;
   for (i = 0; i < 4; i++) {
     xacc = 0;
-    vs_scanline_resample_4tap_AYUV64 ((guint16 *) (tmpbuf + i * dest->stride),
+    vs_scanline_resample_4tap_AYUV64 (tmpbuf + i * dest_pixstride,
         (guint16 *) (src->pixels + CLAMP (i, 0, src->height - 1) * src->stride),
         dest->width, src->width, &xacc, x_increment);
   }
@@ -1478,17 +1480,17 @@ vs_image_scale_4tap_AYUV64 (const VSImage * dest, const VSImage * src,
       k++;
       if (k + 3 < src->height) {
         xacc = 0;
-        vs_scanline_resample_4tap_AYUV64 ((guint16 *) (tmpbuf + ((k +
-                        3) & 3) * dest->stride),
+        vs_scanline_resample_4tap_AYUV64 (tmpbuf + ((k +
+                    3) & 3) * dest_pixstride,
             (guint16 *) (src->pixels + (k + 3) * src->stride), dest->width,
             src->width, &xacc, x_increment);
       }
     }
 
-    t0 = tmpbuf + (CLAMP (j - 1, 0, src->height - 1) & 3) * dest->stride;
-    t1 = tmpbuf + (CLAMP (j, 0, src->height - 1) & 3) * dest->stride;
-    t2 = tmpbuf + (CLAMP (j + 1, 0, src->height - 1) & 3) * dest->stride;
-    t3 = tmpbuf + (CLAMP (j + 2, 0, src->height - 1) & 3) * dest->stride;
+    t0 = tmpbuf + (CLAMP (j - 1, 0, src->height - 1) & 3) * dest_pixstride;
+    t1 = tmpbuf + (CLAMP (j, 0, src->height - 1) & 3) * dest_pixstride;
+    t2 = tmpbuf + (CLAMP (j + 1, 0, src->height - 1) & 3) * dest_pixstride;
+    t3 = tmpbuf + (CLAMP (j + 2, 0, src->height - 1) & 3) * dest_pixstride;
     vs_scanline_merge_4tap_AYUV64 ((guint16 *) (dest->pixels +
             i * dest->stride), t0, t1, t2, t3, dest->width, yacc & 0xffff);
 

tests/check/elements/videoscale.c

@@ -1035,7 +1035,6 @@ videoscale_suite (void)
 #endif
   tcase_add_test (tc_chain, test_basetransform_negotiation);
 
-  GST_ERROR ("FIXME: test 64-bpp formats as well");
   return s;
 }