va: VA-API H.264 decoder and infrastructure

New plugin with an element for H.264 decoding with VA-API. This novel
approach, different from gstreamer-vaapi, uses gstcodecs library for
state handling.

The code is expected to looks cleaner because it uses VA-API without
further layers or wrappers.

* It uses the first supported DRM device as default VA display (other
  displays will be supported through user's GstContext)
* Requires libva >= 1.6
* No multiview/stereo profiles neither interlaced streams because
  gstcodecs doesn't handle them yet
* It is incompatible with gstreamer-vaapi
* Even if memory:VAMemory is exposed, it is not handled yet by any
  other element
* Caps templates are generated dynamically querying VAAPI, but YV12
  and I420 are added for system memory caps because they seem to be
  supported for all the drivers when downloading frames onto main
  memory, as they are used by xvimagesink and others, avoiding color
  conversion.
* Surfaces aren't bounded to context, so they can grow beyond the DBP
  size, allowing smooth reverse playback.
* There isn't yet error handling and recovery.
* 10-bit H.264 streams aren't supported by libva.

Part-of: <!1379>
parent 9ff332e1
Pipeline #168517 waiting for manual action with stages
in 19 seconds
......@@ -151,6 +151,7 @@ option('tinyalsa', type : 'feature', value : 'auto', description : 'TinyALSA plu
option('transcode', type : 'feature', value : 'auto', description : 'Transcode plugin')
option('ttml', type : 'feature', value : 'auto', description : 'TTML subtitle parser and renderer plugin')
option('uvch264', type : 'feature', value : 'auto', description : 'UVC compliant H.264 camera source plugin')
option('va', type : 'feature', value : 'auto', description: 'VA-API new plugin')
option('voaacenc', type : 'feature', value : 'auto', description : 'AAC audio encoder plugin')
option('voamrwbenc', type : 'feature', value : 'auto', description : 'AMR-WB audio encoder plugin')
option('vulkan', type : 'feature', value : 'auto', description : 'Vulkan video sink plugin')
......
......@@ -21,8 +21,8 @@ subdir('shm')
subdir('tinyalsa')
subdir('uvch264')
subdir('v4l2codecs')
subdir('va')
subdir('wasapi')
subdir('wasapi2')
subdir('winks')
subdir('winscreencap')
This diff is collapsed.
/* GStreamer
* Copyright (C) 2020 Igalia, S.L.
* Author: Víctor Jáquez <vjaquez@igalia.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#pragma once
#include <gst/allocators/allocators.h>
#include <gst/video/video.h>
#include "gstvadisplay.h"
G_BEGIN_DECLS
typedef struct _GstVaAllocationParams GstVaAllocationParams;
struct _GstVaAllocationParams
{
GstVideoInfo info;
guint32 usage_hint;
};
#define GST_TYPE_VA_DMABUF_ALLOCATOR (gst_va_dmabuf_allocator_get_type())
G_DECLARE_FINAL_TYPE (GstVaDmabufAllocator, gst_va_dmabuf_allocator, GST,
VA_DMABUF_ALLOCATOR, GstDmaBufAllocator);
GstAllocator * gst_va_dmabuf_allocator_new (GstVaDisplay * display);
gboolean gst_va_dmabuf_setup_buffer (GstAllocator * allocator,
GstBuffer * buffer,
GstVaAllocationParams * params);
gboolean gst_va_dmabuf_try (GstAllocator * allocator,
GstVaAllocationParams * params);
#define GST_TYPE_VA_ALLOCATOR (gst_va_allocator_get_type())
G_DECLARE_FINAL_TYPE (GstVaAllocator, gst_va_allocator, GST, VA_ALLOCATOR, GstAllocator);
#define GST_ALLOCATOR_VASURFACE "VAMemory"
GstAllocator * gst_va_allocator_new (GstVaDisplay * display);
GstMemory * gst_va_allocator_alloc (GstAllocator * allocator,
GstVaAllocationParams * params);
gboolean gst_va_allocator_try (GstAllocator * allocator,
GstVaAllocationParams * params);
VASurfaceID gst_va_memory_get_surface (GstMemory * mem,
GstVideoInfo * info);
VASurfaceID gst_va_buffer_get_surface (GstBuffer * buffer,
GstVideoInfo * info);
G_END_DECLS
This diff is collapsed.
/* GStreamer
* Copyright (C) 2020 Igalia, S.L.
* Author: Víctor Jáquez <vjaquez@igalia.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#pragma once
#include "gstvadisplay.h"
G_BEGIN_DECLS
gboolean gst_va_caps_from_profiles (GstVaDisplay * display,
GArray * profiles,
VAEntrypoint entrypoint,
GstCaps ** codedcaps,
GstCaps ** rawcaps);
VASurfaceAttrib * gst_va_get_surface_attribs (GstVaDisplay * display,
VAConfigID config,
guint * attrib_count);
GstCaps * gst_va_create_raw_caps_from_config (GstVaDisplay * display,
VAConfigID config);
gboolean gst_va_video_format_is_extra (guint format);
G_END_DECLS
This diff is collapsed.
/* GStreamer
* Copyright (C) 2020 Igalia, S.L.
* Author: Víctor Jáquez <vjaquez@igalia.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#pragma once
#include "gstvadisplay.h"
G_BEGIN_DECLS
typedef struct _GstVaDecodePicture GstVaDecodePicture;
struct _GstVaDecodePicture
{
GArray *buffers;
GArray *slices;
VASurfaceID surface;
};
#define GST_TYPE_VA_DECODER (gst_va_decoder_get_type())
G_DECLARE_FINAL_TYPE (GstVaDecoder, gst_va_decoder, GST, VA_DECODER, GstObject)
GstVaDecoder * gst_va_decoder_new (GstVaDisplay * display,
guint32 codec);
gboolean gst_va_decoder_open (GstVaDecoder * self,
VAProfile profile,
guint rt_format);
gboolean gst_va_decoder_close (GstVaDecoder * self);
gboolean gst_va_decoder_is_open (GstVaDecoder * self);
gboolean gst_va_decoder_set_format (GstVaDecoder * self,
gint coded_width,
gint coded_height,
GArray * surfaces);
GstCaps * gst_va_decoder_get_srcpad_caps (GstVaDecoder * self);
GstCaps * gst_va_decoder_get_sinkpad_caps (GstVaDecoder * self);
gboolean gst_va_decoder_has_profile (GstVaDecoder * self,
VAProfile profile);
gint gst_va_decoder_get_mem_types (GstVaDecoder * self);
gboolean gst_va_decoder_add_param_buffer (GstVaDecoder * self,
GstVaDecodePicture * pic,
gint type,
gpointer data,
gsize size);
gboolean gst_va_decoder_add_slice_buffer (GstVaDecoder * self,
GstVaDecodePicture * pic,
gpointer params_data,
gsize params_size,
gpointer slice_data,
gsize slice_size);
gboolean gst_va_decoder_decode (GstVaDecoder * self,
GstVaDecodePicture * pic);
gboolean gst_va_decoder_destroy_buffers (GstVaDecoder * self,
GstVaDecodePicture * pic);
GstVaDecodePicture * gst_va_decoder_new_decode_picture (GstVaDecoder * self,
VASurfaceID surface);
void gst_va_decode_picture_free (GstVaDecodePicture * pic);
G_END_DECLS
/* GStreamer
* Copyright (C) 2020 Igalia, S.L.
* Author: Víctor Jáquez <vjaquez@igalia.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstvadevice.h"
#include <gudev/gudev.h>
#include "gstvadisplay_drm.h"
#define GST_CAT_DEFAULT gstva_debug
GST_DEBUG_CATEGORY_EXTERN (gstva_debug);
GST_DEFINE_MINI_OBJECT_TYPE (GstVaDevice, gst_va_device);
static void
gst_va_device_free (GstVaDevice * device)
{
if (device->display)
gst_object_unref (device->display);
g_free (device->render_device_path);
g_free (device);
}
static GstVaDevice *
gst_va_device_new (GstVaDisplay * display, const gchar * render_device_path)
{
GstVaDevice *device = g_new0 (GstVaDevice, 1);
gst_mini_object_init (GST_MINI_OBJECT_CAST (device), 0, GST_TYPE_VA_DEVICE,
NULL, NULL, (GstMiniObjectFreeFunction) gst_va_device_free);
/* take ownership */
device->display = display;
device->render_device_path = g_strdup (render_device_path);
return device;
}
GList *
gst_va_device_find_devices (void)
{
GUdevClient *client;
GList *udev_devices, *dev;
GQueue devices = G_QUEUE_INIT;
client = g_udev_client_new (NULL);
udev_devices = g_udev_client_query_by_subsystem (client, "drm");
for (dev = udev_devices; dev; dev = g_list_next (dev)) {
GstVaDisplay *dpy;
GUdevDevice *udev = (GUdevDevice *) dev->data;
const gchar *path = g_udev_device_get_device_file (udev);
const gchar *name = g_udev_device_get_name (udev);
if (!path || !g_str_has_prefix (name, "renderD")) {
GST_LOG ("Ignoring %s in %s", name, path);
continue;
}
if (!(dpy = gst_va_display_drm_new_from_path (path)))
continue;
GST_INFO ("Found VA-API device: %s", path);
g_queue_push_tail (&devices, gst_va_device_new (dpy, path));
}
g_list_free_full (udev_devices, g_object_unref);
g_object_unref (client);
return devices.head;
}
void
gst_va_device_list_free (GList * devices)
{
g_list_free_full (devices, (GDestroyNotify) gst_mini_object_unref);
}
/* GStreamer
* Copyright (C) 2020 Igalia, S.L.
* Author: Víctor Jáquez <vjaquez@igalia.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#pragma once
#include <gst/gst.h>
G_BEGIN_DECLS
#include "gstvadisplay.h"
#define GST_TYPE_VA_DEVICE (gst_va_device_get_type())
#define GST_IS_VA_DEVICE(obj) (GST_IS_MINI_OBJECT_TYPE((obj), GST_TYPE_VA_DEVICE))
#define GST_VA_DEVICE(obj) ((GstVaDevice *)(obj))
typedef struct
{
GstMiniObject mini_object;
GstVaDisplay *display;
gchar *render_device_path;
} GstVaDevice;
GType gst_va_device_get_type (void);
GList * gst_va_device_find_devices (void);
void gst_va_device_list_free (GList * devices);
G_END_DECLS
/* GStreamer
* Copyright (C) 2020 Igalia, S.L.
* Author: Víctor Jáquez <vjaquez@igalia.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstvadisplay.h"
#include "gstvaprofile.h"
GST_DEBUG_CATEGORY (gst_va_display_debug);
#define GST_CAT_DEFAULT gst_va_display_debug
typedef struct _GstVaDisplayPrivate GstVaDisplayPrivate;
struct _GstVaDisplayPrivate
{
GRecMutex lock;
VADisplay display;
gboolean foreign;
gboolean init;
};
#define gst_va_display_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstVaDisplay, gst_va_display, GST_TYPE_OBJECT,
G_ADD_PRIVATE (GstVaDisplay);
GST_DEBUG_CATEGORY_INIT (gst_va_display_debug, "vadisplay", 0,
"VA Display"));
enum
{
PROP_VA_DISPLAY = 1,
N_PROPERTIES
};
static GParamSpec *g_properties[N_PROPERTIES];
#define GET_PRIV(obj) gst_va_display_get_instance_private (GST_VA_DISPLAY (obj))
static gboolean
_gst_va_display_driver_filter (VADisplay display)
{
const char *vendor = vaQueryVendorString (display);
GST_INFO ("VA-API driver vendor: %s", vendor);
/* XXX(victor): driver quirks & driver whitelist */
return TRUE;
}
static void
gst_va_display_set_display (GstVaDisplay * self, gpointer display)
{
GstVaDisplayPrivate *priv = GET_PRIV (self);
if (!display)
return;
if (vaDisplayIsValid (display) == 0) {
GST_WARNING_OBJECT (self,
"User's VA display is invalid. An internal one will be tried.");
return;
}
if (!_gst_va_display_driver_filter (display))
return;
priv->display = display;
priv->foreign = TRUE;
/* assume driver is already initialized */
priv->init = TRUE;
}
static void
gst_va_display_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstVaDisplay *self = GST_VA_DISPLAY (object);
switch (prop_id) {
case PROP_VA_DISPLAY:{
gpointer display = g_value_get_pointer (value);
gst_va_display_set_display (self, display);
break;
}
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_va_display_get_property (GObject * object, guint prop_id, GValue * value,
GParamSpec * pspec)
{
GstVaDisplayPrivate *priv = GET_PRIV (object);
switch (prop_id) {
case PROP_VA_DISPLAY:
g_value_set_pointer (value, priv->display);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_va_display_constructed (GObject * object)
{
GstVaDisplay *self = GST_VA_DISPLAY (object);
GstVaDisplayPrivate *priv = GET_PRIV (object);
GstVaDisplayClass *klass = GST_VA_DISPLAY_GET_CLASS (object);
if (!priv->display && klass->create_va_display)
priv->display = klass->create_va_display (self);
G_OBJECT_CLASS (parent_class)->constructed (object);
}
static void
gst_va_display_dispose (GObject * object)
{
GstVaDisplayPrivate *priv = GET_PRIV (object);
if (priv->display && !priv->foreign)
vaTerminate (priv->display);
priv->display = NULL;
G_OBJECT_CLASS (parent_class)->dispose (object);
}
static void
gst_va_display_finalize (GObject * object)
{
GstVaDisplayPrivate *priv = GET_PRIV (object);
g_rec_mutex_clear (&priv->lock);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static void
gst_va_display_class_init (GstVaDisplayClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
gobject_class->set_property = gst_va_display_set_property;
gobject_class->get_property = gst_va_display_get_property;
gobject_class->constructed = gst_va_display_constructed;
gobject_class->dispose = gst_va_display_dispose;
gobject_class->finalize = gst_va_display_finalize;
g_properties[PROP_VA_DISPLAY] =
g_param_spec_pointer ("va-display", "VADisplay", "VA Display handler",
G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS);
g_object_class_install_properties (gobject_class, N_PROPERTIES, g_properties);
}
static void
gst_va_display_init (GstVaDisplay * self)
{
GstVaDisplayPrivate *priv = GET_PRIV (self);
g_rec_mutex_init (&priv->lock);
}
void
gst_va_display_lock (GstVaDisplay * self)
{
GstVaDisplayPrivate *priv;
g_return_if_fail (GST_IS_VA_DISPLAY (self));
priv = GET_PRIV (self);
g_rec_mutex_lock (&priv->lock);
}
void
gst_va_display_unlock (GstVaDisplay * self)
{
GstVaDisplayPrivate *priv;
g_return_if_fail (GST_IS_VA_DISPLAY (self));
priv = GET_PRIV (self);
g_rec_mutex_unlock (&priv->lock);
}
#ifndef GST_DISABLE_GST_DEBUG
static gchar *
_strip_msg (const char *message)
{
gchar *msg = g_strdup (message);
if (!msg)
return NULL;
return g_strstrip (msg);
}
static void
_va_warning (gpointer object, const char *message)
{
GstVaDisplay *self = GST_VA_DISPLAY (object);
gchar *msg;
if ((msg = _strip_msg (message))) {
GST_WARNING_OBJECT (self, "VA error: %s", msg);
g_free (msg);
}
}
static void
_va_info (gpointer object, const char *message)
{
GstVaDisplay *self = GST_VA_DISPLAY (object);
gchar *msg;
if ((msg = _strip_msg (message))) {
GST_INFO_OBJECT (self, "VA info: %s", msg);
g_free (msg);
}
}
#endif
/**
* gst_va_display_initialize:
* @self: a #GstVaDisplay
*
* If the display is set by the user (foreign) it is assumed that the
* driver is already initialized, thus this function is noop.
*
* If the display is opened internally, this function will initialize
* the driver and it will set driver's message callbacks.
*
* NOTE: this function is supposed to be private, only used by
* GstVaDisplay descendants.
*
* Returns: %TRUE if the VA driver can be initialized; %FALSE
* otherwise
**/
gboolean
gst_va_display_initialize (GstVaDisplay * self)
{
GstVaDisplayPrivate *priv;
VAStatus status;
int major_version = -1, minor_version = -1;
g_return_val_if_fail (GST_IS_VA_DISPLAY (self), FALSE);
priv = GET_PRIV (self);
if (priv->init)
return TRUE;
if (!priv->display)
return FALSE;
#ifndef GST_DISABLE_GST_DEBUG
vaSetErrorCallback (priv->display, _va_warning, self);
vaSetInfoCallback (priv->display, _va_info, self);
#endif
status = vaInitialize (priv->display, &major_version, &minor_version);
if (status != VA_STATUS_SUCCESS) {
GST_ERROR_OBJECT (self, "vaInitialize: %s", vaErrorStr (status));
return FALSE;
}
GST_INFO_OBJECT (self, "VA-API version %d.%d", major_version, minor_version);
priv->init = TRUE;
if (!_gst_va_display_driver_filter (priv->display))
return FALSE;
return TRUE;
}
VADisplay
gst_va_display_get_va_dpy (GstVaDisplay * self)
{
VADisplay dpy = 0;
g_return_val_if_fail (GST_IS_VA_DISPLAY (self), 0);
g_object_get (self, "va-display", &dpy, NULL);
return dpy;
}
GArray *
gst_va_display_get_profiles (GstVaDisplay * self, guint32 codec,
VAEntrypoint entrypoint)
{
GArray *ret = NULL;
VADisplay dpy;
VAEntrypoint *entrypoints;
VAProfile *profiles;
VAStatus status;
gint i, j, num_entrypoints = 0, num_profiles = 0;
g_return_val_if_fail (GST_IS_VA_DISPLAY (self), NULL);
dpy = gst_va_display_get_va_dpy (self);
gst_va_display_lock (self);
num_profiles = vaMaxNumProfiles (dpy);
num_entrypoints = vaMaxNumEntrypoints (dpy);
gst_va_display_unlock (self);
profiles = g_new (VAProfile, num_profiles);
entrypoints = g_new (VAEntrypoint, num_entrypoints);
gst_va_display_lock (self);
status = vaQueryConfigProfiles (dpy, profiles, &num_profiles);
gst_va_display_unlock (self);
if (status != VA_STATUS_SUCCESS) {
GST_ERROR ("vaQueryConfigProfile: %s", vaErrorStr (status));