overlay: fix build without advanced GstVideoOverlayFormatFlags.

Check for global-alpha support in GstVideoOverlayComposition API.
Signed-off-by: Gwenole Beauchesne <gwenole.beauchesne@intel.com>

configure.ac

@@ -200,7 +200,8 @@ AC_CACHE_CHECK([for GstVideoOverlayComposition],
     AC_COMPILE_IFELSE(
         [AC_LANG_PROGRAM(
             [[#include <gst/video/video-overlay-composition.h>]],
-            [[GstVideoOverlayComposition *c = gst_video_overlay_composition_new(0);]])],
+            [[GstVideoOverlayComposition *c;
+              c = gst_video_overlay_composition_new(0);]])],
         [ac_cv_have_gst_video_overlay_composition="yes"],
         [ac_cv_have_gst_video_overlay_composition="no"]
     )
@@ -211,6 +212,34 @@ if test "$ac_cv_have_gst_video_overlay_composition" != "yes"; then
     AC_MSG_ERROR([GstVideoOverlayComposition is not available])
 fi
 
+AC_CACHE_CHECK([... with advanced format flags],
+    ac_cv_have_gst_video_overlay_hwcaps, [
+    saved_CPPFLAGS="$CPPFLAGS"
+    CPPFLAGS="$CPPFLAGS $GST_CFLAGS $GST_VIDEO_CFLAGS"
+    saved_LIBS="$LIBS"
+    LIBS="$LIBS $GST_LIBS $GST_VIDEO_LIBS"
+    AC_COMPILE_IFELSE(
+        [AC_LANG_PROGRAM(
+            [[#include <gst/video/video-overlay-composition.h>]],
+            [[GstVideoOverlayRectangle *rect;
+              const guint w = 64, h = 64;
+              guint flags = (
+                  GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA|
+                  GST_VIDEO_OVERLAY_FORMAT_FLAG_GLOBAL_ALPHA);
+              rect = gst_video_overlay_rectangle_new_argb(NULL, w, h, w * 4,
+                  0, 0, w, h, flags);
+              gst_video_overlay_rectangle_set_global_alpha(rect, 0.5f);]])],
+        [ac_cv_have_gst_video_overlay_hwcaps="yes"],
+        [ac_cv_have_gst_video_overlay_hwcaps="no"]
+    )
+    CPPFLAGS="$saved_CPPFLAGS"
+    LIBS="$saved_LIBS"
+])
+if test "$ac_cv_have_gst_video_overlay_hwcaps" = "yes"; then
+    AC_DEFINE_UNQUOTED([HAVE_GST_VIDEO_OVERLAY_HWCAPS], 1,
+        [Defined to 1 if GstVideoOverlayComposition API supports HW hints.])
+fi
+
 dnl ... GstVideoDecoder (gstreamer-video)
 AC_CACHE_CHECK([for GstVideoDecoder],
     ac_cv_have_gst_video_decoder, [

gst-libs/gst/vaapi/gstvaapicontext.c

@@ -295,9 +295,11 @@ static inline gboolean
 overlay_rectangle_update_global_alpha(GstVaapiOverlayRectangle *overlay,
     GstVideoOverlayRectangle *rect)
 {
+#ifdef HAVE_GST_VIDEO_OVERLAY_HWCAPS
     const guint flags = gst_video_overlay_rectangle_get_flags(rect);
     if (!(flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_GLOBAL_ALPHA))
         return TRUE;
+#endif
     return gst_vaapi_subpicture_set_global_alpha(overlay->subpicture,
         gst_video_overlay_rectangle_get_global_alpha(rect));
 }

gst-libs/gst/vaapi/gstvaapiutils.c

@@ -260,10 +260,12 @@ from_GstVideoOverlayFormatFlags(guint ovl_flags)
 {
     guint flags = 0;
 
+#ifdef HAVE_GST_VIDEO_OVERLAY_HWCAPS
     if (ovl_flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA)
         flags |= GST_VAAPI_SUBPICTURE_FLAG_PREMULTIPLIED_ALPHA;
     if (ovl_flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_GLOBAL_ALPHA)
         flags |= GST_VAAPI_SUBPICTURE_FLAG_GLOBAL_ALPHA;
+#endif
     return flags;
 }
 
@@ -280,10 +282,12 @@ to_GstVideoOverlayFormatFlags(guint flags)
 {
     guint ovl_flags = 0;
 
+#ifdef HAVE_GST_VIDEO_OVERLAY_HWCAPS
     if (flags & GST_VAAPI_SUBPICTURE_FLAG_PREMULTIPLIED_ALPHA)
         ovl_flags |= GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA;
     if (flags & GST_VAAPI_SUBPICTURE_FLAG_GLOBAL_ALPHA)
         ovl_flags |= GST_VIDEO_OVERLAY_FORMAT_FLAG_GLOBAL_ALPHA;
+#endif
     return ovl_flags;
 }
 

gst-libs/gst/vaapi/sysdeps.h

@@ -28,4 +28,10 @@
 
 #include "glibcompat.h"
 
+/* <gst/video/video-overlay-composition.h> compatibility glue */
+#ifndef HAVE_GST_VIDEO_OVERLAY_HWCAPS
+# define gst_video_overlay_rectangle_get_flags(rect) (0)
+# define gst_video_overlay_rectangle_get_global_alpha(rect) (1.0f)
+#endif
+
 #endif /* SYSDEPS_H */

tests/test-subpicture.c

@@ -41,10 +41,12 @@ static GOptionEntry g_options[] = {
       0,
       G_OPTION_ARG_STRING, &g_codec_str,
       "codec to test", NULL },
+#ifdef HAVE_GST_VIDEO_OVERLAY_HWCAPS
     { "global-alpha", 'g',
       0,
       G_OPTION_ARG_DOUBLE, &g_global_alpha,
       "global-alpha value", NULL },
+#endif
     { NULL, }
 };
 
@@ -82,8 +84,10 @@ main(int argc, char *argv[])
     if (!video_output_init(&argc, argv, g_options))
         g_error("failed to initialize video output subsystem");
 
+#ifdef HAVE_GST_VIDEO_OVERLAY_HWCAPS
     if (g_global_alpha != 1.0)
         flags |= GST_VIDEO_OVERLAY_FORMAT_FLAG_GLOBAL_ALPHA;
+#endif
 
     g_print("Test subpicture\n");
 
@@ -123,8 +127,10 @@ main(int argc, char *argv[])
         g_error("could not create video overlay");
     gst_buffer_unref(buffer);
 
+#ifdef HAVE_GST_VIDEO_OVERLAY_HWCAPS
     if (flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_GLOBAL_ALPHA)
         gst_video_overlay_rectangle_set_global_alpha(overlay, g_global_alpha);
+#endif
 
     compo = gst_video_overlay_composition_new(overlay);
     if (!compo)