Commit 7892e1e2 authored by Tim-Philipp Müller's avatar Tim-Philipp Müller 🐠

videoscale: fix 4-tap scaling for 64-bpp formats

Fix invalid memory access caused by broken pointer arithmetic.

If we have a uint16_t *tmpbuf and add n * dest->stride to it, we
skip twice as much as we intended to because dest->stride is in
bytes and not in pixels. This made us write beyond the end of
our allocated temp buffer, and made the unit test crash.
parent a3143250
......@@ -532,7 +532,7 @@ gst_video_scale_set_info (GstVideoFilter * filter, GstCaps * in,
if (videoscale->tmp_buf)
g_free (videoscale->tmp_buf);
videoscale->tmp_buf = g_malloc (out_info->width * 8 * 4);
videoscale->tmp_buf = g_malloc (out_info->width * sizeof (guint64) * 4);
if (in_info->width == out_info->width && in_info->height == out_info->height) {
gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (filter), TRUE);
......
......@@ -1449,6 +1449,8 @@ vs_image_scale_4tap_AYUV64 (const VSImage * dest, const VSImage * src,
int xacc;
int k;
guint16 *tmpbuf = (guint16 *) tmpbuf8;
/* destination stride in pixels for easier use with tmpbuf variable */
int dest_pixstride = dest->stride / sizeof (guint16);
if (dest->height == 1)
y_increment = 0;
......@@ -1463,7 +1465,7 @@ vs_image_scale_4tap_AYUV64 (const VSImage * dest, const VSImage * src,
k = 0;
for (i = 0; i < 4; i++) {
xacc = 0;
vs_scanline_resample_4tap_AYUV64 ((guint16 *) (tmpbuf + i * dest->stride),
vs_scanline_resample_4tap_AYUV64 (tmpbuf + i * dest_pixstride,
(guint16 *) (src->pixels + CLAMP (i, 0, src->height - 1) * src->stride),
dest->width, src->width, &xacc, x_increment);
}
......@@ -1478,17 +1480,17 @@ vs_image_scale_4tap_AYUV64 (const VSImage * dest, const VSImage * src,
k++;
if (k + 3 < src->height) {
xacc = 0;
vs_scanline_resample_4tap_AYUV64 ((guint16 *) (tmpbuf + ((k +
3) & 3) * dest->stride),
vs_scanline_resample_4tap_AYUV64 (tmpbuf + ((k +
3) & 3) * dest_pixstride,
(guint16 *) (src->pixels + (k + 3) * src->stride), dest->width,
src->width, &xacc, x_increment);
}
}
t0 = tmpbuf + (CLAMP (j - 1, 0, src->height - 1) & 3) * dest->stride;
t1 = tmpbuf + (CLAMP (j, 0, src->height - 1) & 3) * dest->stride;
t2 = tmpbuf + (CLAMP (j + 1, 0, src->height - 1) & 3) * dest->stride;
t3 = tmpbuf + (CLAMP (j + 2, 0, src->height - 1) & 3) * dest->stride;
t0 = tmpbuf + (CLAMP (j - 1, 0, src->height - 1) & 3) * dest_pixstride;
t1 = tmpbuf + (CLAMP (j, 0, src->height - 1) & 3) * dest_pixstride;
t2 = tmpbuf + (CLAMP (j + 1, 0, src->height - 1) & 3) * dest_pixstride;
t3 = tmpbuf + (CLAMP (j + 2, 0, src->height - 1) & 3) * dest_pixstride;
vs_scanline_merge_4tap_AYUV64 ((guint16 *) (dest->pixels +
i * dest->stride), t0, t1, t2, t3, dest->width, yacc & 0xffff);
......
......@@ -1035,7 +1035,6 @@ videoscale_suite (void)
#endif
tcase_add_test (tc_chain, test_basetransform_negotiation);
GST_ERROR ("FIXME: test 64-bpp formats as well");
return s;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment