Commit b08b8dda authored by Hyunjun Ko's avatar Hyunjun Ko Committed by Sreerenj Balachandran

msdk: manage MSDK surfaces seperately

Currently a gst buffer has one mfxFrameSurface when it's allocated and
can't be changed.
This is based on that the life of gst buffer and mfxFrameSurface would
be same.
But it's not true. Sometimes even if a gst buffer of a frame is finished
on downstream,
mfxFramesurface coupled with the gst buffer is still locked, which means
it's still being used in the driver.

So this patch does this.
Every time a gst buffer is acquired from the pool, it confirms if the
surface coupled with the buffer is unlocked.
If not, replace it with new unlocked one.
In this way, user(decoder or encoder) doesn't need to manage gst buffers
including locked surface.

To do that, this patch includes the following:
1. GstMsdkContext
- Manages MSDK surfaces available, used, locked respectively as the
following:
  1\ surfaces_avail : surfaces which are free and unused anywhere
  2\ surfaces_used : surfaces coupled with a gst buffer and being used
now.
  3\ surfaces_locked : surfaces still locked even after the gst buffer
is released.

- Provide an api to get MSDK surface available.
- Provide an api to release MSDK surface.

2. GstMsdkVideoMemory
- Gets a surface available when it's allocated.
- Provide an api to get an available surface with new unlocked one.
- Provide an api to release surface in the msdk video memory.

3. GstMsdkBufferPool
- In acquire_buffer, every time a gst buffer is acquired, get new
available surface from the list.
- In release_buffer, it confirms if the buffer's surface is unlocked or
not.
  - If unlocked, it is put to the available list.
  - If still locked, it is put to the locked list.

This also fixes bug #793525.

https://bugzilla.gnome.org/show_bug.cgi?id=793413
https://bugzilla.gnome.org/show_bug.cgi?id=793525
parent d156163c
......@@ -227,6 +227,64 @@ no_memory:
}
}
static GstFlowReturn
gst_msdk_buffer_pool_acquire_buffer (GstBufferPool * pool,
GstBuffer ** out_buffer_ptr, GstBufferPoolAcquireParams * params)
{
GstMsdkBufferPool *msdk_pool = GST_MSDK_BUFFER_POOL_CAST (pool);
GstMsdkBufferPoolPrivate *priv = msdk_pool->priv;
GstBuffer *buf = NULL;
GstFlowReturn ret;
mfxFrameSurface1 *surface;
ret =
GST_BUFFER_POOL_CLASS (parent_class)->acquire_buffer (pool, &buf, params);
/* When using video memory, mfx surface is still locked even though
* it's finished by SyncOperation. There's no way to get notified when it gets unlocked.
* So we need to confirm if it's unlocked every time a gst buffer is acquired.
* If it's still locked, we can replace it with new unlocked/unused surface.
*/
if (ret != GST_FLOW_OK || !priv->use_video_memory) {
if (buf)
*out_buffer_ptr = buf;
return ret;
}
surface = gst_msdk_get_surface_from_buffer (buf);
if (!surface || surface->Data.Locked > 0) {
if (!gst_msdk_video_memory_get_surface_available (GST_MSDK_VIDEO_MEMORY_CAST
(gst_buffer_peek_memory (buf, 0)))) {
GST_WARNING_OBJECT (pool, "failed to get new surface available");
return GST_FLOW_ERROR;
}
}
*out_buffer_ptr = buf;
return GST_FLOW_OK;
}
static void
gst_msdk_buffer_pool_release_buffer (GstBufferPool * pool, GstBuffer * buf)
{
mfxFrameSurface1 *surface;
GstMsdkBufferPool *msdk_pool = GST_MSDK_BUFFER_POOL_CAST (pool);
GstMsdkBufferPoolPrivate *priv = msdk_pool->priv;
if (!priv->use_video_memory)
goto done;
surface = gst_msdk_get_surface_from_buffer (buf);
if (!surface)
goto done;
gst_msdk_video_memory_release_surface (GST_MSDK_VIDEO_MEMORY_CAST
(gst_buffer_peek_memory (buf, 0)));
done:
return GST_BUFFER_POOL_CLASS (parent_class)->release_buffer (pool, buf);
}
static void
gst_msdk_buffer_pool_finalize (GObject * object)
{
......@@ -258,6 +316,8 @@ gst_msdk_buffer_pool_class_init (GstMsdkBufferPoolClass * klass)
pool_class->get_options = gst_msdk_buffer_pool_get_options;
pool_class->set_config = gst_msdk_buffer_pool_set_config;
pool_class->alloc_buffer = gst_msdk_buffer_pool_alloc_buffer;
pool_class->acquire_buffer = gst_msdk_buffer_pool_acquire_buffer;
pool_class->release_buffer = gst_msdk_buffer_pool_release_buffer;
}
GstBufferPool *
......
......@@ -58,6 +58,7 @@ struct _GstMsdkContextPrivate
gboolean is_joined;
GstMsdkContextJobType job_type;
gint shared_async_depth;
GMutex mutex;
#ifndef _WIN32
gint fd;
VADisplay dpy;
......@@ -208,6 +209,8 @@ gst_msdk_context_init (GstMsdkContext * context)
GstMsdkContextPrivate *priv = GST_MSDK_CONTEXT_GET_PRIVATE (context);
context->priv = priv;
g_mutex_init (&priv->mutex);
}
static void
......@@ -222,6 +225,7 @@ gst_msdk_context_finalize (GObject * obj)
}
msdk_close_session (priv->session);
g_mutex_clear (&priv->mutex);
#ifndef _WIN32
if (priv->dpy)
......@@ -365,12 +369,47 @@ gst_msdk_context_get_cached_alloc_responses_by_request (GstMsdkContext *
return NULL;
}
static void
create_surfaces (GstMsdkContext * context, GstMsdkAllocResponse * resp)
{
gint i;
mfxMemId *mem_id;
mfxFrameSurface1 *surface;
for (i = 0; i < resp->response->NumFrameActual; i++) {
mem_id = resp->mem_ids[i];
surface = (mfxFrameSurface1 *) g_slice_new0 (mfxFrameSurface1);
if (!surface) {
GST_ERROR ("failed to allocate surface");
break;
}
surface->Data.MemId = mem_id;
resp->surfaces_avail = g_list_prepend (resp->surfaces_avail, surface);
}
}
static void
free_surface (gpointer surface)
{
g_slice_free1 (sizeof (mfxFrameSurface1), surface);
}
static void
remove_surfaces (GstMsdkContext * context, GstMsdkAllocResponse * resp)
{
g_list_free_full (resp->surfaces_used, free_surface);
g_list_free_full (resp->surfaces_avail, free_surface);
g_list_free_full (resp->surfaces_locked, free_surface);
}
void
gst_msdk_context_add_alloc_response (GstMsdkContext * context,
GstMsdkAllocResponse * resp)
{
context->priv->cached_alloc_responses =
g_list_append (context->priv->cached_alloc_responses, resp);
g_list_prepend (context->priv->cached_alloc_responses, resp);
create_surfaces (context, resp);
}
gboolean
......@@ -387,6 +426,8 @@ gst_msdk_context_remove_alloc_response (GstMsdkContext * context,
msdk_resp = l->data;
remove_surfaces (context, msdk_resp);
g_slice_free1 (sizeof (GstMsdkAllocResponse), msdk_resp);
priv->cached_alloc_responses =
g_list_delete_link (priv->cached_alloc_responses, l);
......@@ -394,6 +435,121 @@ gst_msdk_context_remove_alloc_response (GstMsdkContext * context,
return TRUE;
}
static gboolean
check_surfaces_available (GstMsdkContext * context, GstMsdkAllocResponse * resp)
{
GList *l;
mfxFrameSurface1 *surface = NULL;
GstMsdkContextPrivate *priv = context->priv;
gboolean ret = FALSE;
g_mutex_lock (&priv->mutex);
for (l = resp->surfaces_locked; l; l = l->next) {
surface = l->data;
if (!surface->Data.Locked) {
resp->surfaces_locked = g_list_remove (resp->surfaces_locked, surface);
resp->surfaces_avail = g_list_prepend (resp->surfaces_avail, surface);
ret = TRUE;
}
}
g_mutex_unlock (&priv->mutex);
return ret;
}
/*
* There are 3 lists here in GstMsdkContext as the following:
* 1. surfaces_avail : surfaces which are free and unused anywhere
* 2. surfaces_used : surfaces coupled with a gst buffer and being used now.
* 3. surfaces_locked : surfaces still locked even after the gst buffer is released.
*
* Note that they need to be protected by mutex to be thread-safe.
*/
mfxFrameSurface1 *
gst_msdk_context_get_surface_available (GstMsdkContext * context,
mfxFrameAllocResponse * resp)
{
GList *l;
mfxFrameSurface1 *surface = NULL;
GstMsdkAllocResponse *msdk_resp =
gst_msdk_context_get_cached_alloc_responses (context, resp);
gint retry = 0;
GstMsdkContextPrivate *priv = context->priv;
retry:
g_mutex_lock (&priv->mutex);
for (l = msdk_resp->surfaces_avail; l; l = l->next) {
surface = l->data;
if (!surface->Data.Locked) {
msdk_resp->surfaces_avail =
g_list_remove (msdk_resp->surfaces_avail, surface);
msdk_resp->surfaces_used =
g_list_prepend (msdk_resp->surfaces_used, surface);
break;
}
}
g_mutex_unlock (&priv->mutex);
/*
* If a msdk context is shared by multiple msdk elements,
* upstream msdk element sometimes needs to wait for a gst buffer
* to be released in downstream.
*
* Poll the pool for a maximum of 20 milisecnds.
*
* FIXME: Is there any better way to handle this case?
*/
if (!surface && retry < 20) {
/* If there's no surface available, find unlocked surfaces in the locked list,
* take it back to the available list and then search again.
*/
check_surfaces_available (context, msdk_resp);
retry++;
g_usleep (1000);
goto retry;
}
return surface;
}
void
gst_msdk_context_put_surface_locked (GstMsdkContext * context,
mfxFrameAllocResponse * resp, mfxFrameSurface1 * surface)
{
GstMsdkContextPrivate *priv = context->priv;
GstMsdkAllocResponse *msdk_resp =
gst_msdk_context_get_cached_alloc_responses (context, resp);
g_mutex_lock (&priv->mutex);
if (!g_list_find (msdk_resp->surfaces_locked, surface)) {
msdk_resp->surfaces_used =
g_list_remove (msdk_resp->surfaces_used, surface);
msdk_resp->surfaces_locked =
g_list_prepend (msdk_resp->surfaces_locked, surface);
}
g_mutex_unlock (&priv->mutex);
}
void
gst_msdk_context_put_surface_available (GstMsdkContext * context,
mfxFrameAllocResponse * resp, mfxFrameSurface1 * surface)
{
GstMsdkContextPrivate *priv = context->priv;
GstMsdkAllocResponse *msdk_resp =
gst_msdk_context_get_cached_alloc_responses (context, resp);
g_mutex_lock (&priv->mutex);
if (!g_list_find (msdk_resp->surfaces_avail, surface)) {
msdk_resp->surfaces_used =
g_list_remove (msdk_resp->surfaces_used, surface);
msdk_resp->surfaces_avail =
g_list_prepend (msdk_resp->surfaces_avail, surface);
}
g_mutex_unlock (&priv->mutex);
}
GstMsdkContextJobType
gst_msdk_context_get_job_type (GstMsdkContext * context)
{
......
......@@ -100,7 +100,9 @@ struct _GstMsdkAllocResponse {
mfxFrameAllocResponse *response;
mfxFrameAllocRequest request;
mfxMemId *mem_ids;
gint num_used_memory;
GList *surfaces_avail;
GList *surfaces_used;
GList *surfaces_locked;
};
GstMsdkAllocResponse *
......@@ -119,6 +121,15 @@ gboolean
gst_msdk_context_remove_alloc_response (GstMsdkContext * context,
mfxFrameAllocResponse * resp);
mfxFrameSurface1 *
gst_msdk_context_get_surface_available (GstMsdkContext * context, mfxFrameAllocResponse * resp);
void
gst_msdk_context_put_surface_locked (GstMsdkContext * context, mfxFrameAllocResponse * resp, mfxFrameSurface1 * surface);
void
gst_msdk_context_put_surface_available (GstMsdkContext * context, mfxFrameAllocResponse * resp, mfxFrameSurface1 * surface);
GstMsdkContextJobType
gst_msdk_context_get_job_type (GstMsdkContext * context);
......
......@@ -36,37 +36,19 @@
#include "gstmsdkvideomemory.h"
#include "gstmsdkallocator.h"
static gboolean
ensure_data (GstMsdkVideoMemory * mem, GstMsdkVideoAllocator * allocator)
{
GstMsdkMemoryID *mem_id;
GstMsdkAllocResponse *resp =
gst_msdk_context_get_cached_alloc_responses (allocator->context,
allocator->alloc_response);
if (!resp) {
GST_WARNING ("failed to get allocation response");
return FALSE;
}
mem_id = (GstMsdkMemoryID *) resp->mem_ids[resp->num_used_memory++];
mem->surface->Data.MemId = mem_id;
return TRUE;
}
static mfxFrameSurface1 *
gst_msdk_video_allocator_create_surface (GstAllocator * allocator)
gst_msdk_video_allocator_get_surface (GstAllocator * allocator)
{
mfxFrameInfo frame_info = { {0,}, 0, };
mfxFrameSurface1 *surface;
GstMsdkVideoAllocator *msdk_video_allocator =
GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
surface = (mfxFrameSurface1 *) g_slice_new0 (mfxFrameSurface1);
surface =
gst_msdk_context_get_surface_available (msdk_video_allocator->context,
msdk_video_allocator->alloc_response);
if (!surface) {
GST_ERROR ("failed to allocate surface");
GST_ERROR ("failed to get surface available");
return NULL;
}
......@@ -78,6 +60,42 @@ gst_msdk_video_allocator_create_surface (GstAllocator * allocator)
return surface;
}
gboolean
gst_msdk_video_memory_get_surface_available (GstMsdkVideoMemory * mem)
{
GstAllocator *allocator;
allocator = GST_MEMORY_CAST (mem)->allocator;
mem->surface = gst_msdk_video_allocator_get_surface (allocator);
return mem->surface ? TRUE : FALSE;
}
/*
* Every time releasing a gst buffer, we need to check the status of surface's lock,
* so that we could manage locked surfaces seperatedly in the context.
* Otherwise, we put the surface to the available list.
*/
void
gst_msdk_video_memory_release_surface (GstMsdkVideoMemory * mem)
{
GstMsdkVideoAllocator *msdk_video_allocator;
msdk_video_allocator =
GST_MSDK_VIDEO_ALLOCATOR_CAST (GST_MEMORY_CAST (mem)->allocator);
if (!mem->surface)
return;
if (mem->surface->Data.Locked > 0)
gst_msdk_context_put_surface_locked (msdk_video_allocator->context,
msdk_video_allocator->alloc_response, mem->surface);
else
gst_msdk_context_put_surface_available (msdk_video_allocator->context,
msdk_video_allocator->alloc_response, mem->surface);
mem->surface = NULL;
return;
}
GstMemory *
gst_msdk_video_memory_new (GstAllocator * base_allocator)
{
......@@ -94,16 +112,15 @@ gst_msdk_video_memory_new (GstAllocator * base_allocator)
if (!mem)
return NULL;
mem->surface = gst_msdk_video_allocator_create_surface (base_allocator);
mem->surface = gst_msdk_video_allocator_get_surface (base_allocator);
if (!mem->surface)
return FALSE;
vip = &allocator->image_info;
gst_memory_init (&mem->parent_instance, GST_MEMORY_FLAG_NO_SHARE,
base_allocator, NULL, GST_VIDEO_INFO_SIZE (vip), 0, 0,
GST_VIDEO_INFO_SIZE (vip));
if (!ensure_data (mem, allocator))
return FALSE;
return GST_MEMORY_CAST (mem);
}
......@@ -242,19 +259,6 @@ gst_msdk_video_allocator_alloc (GstAllocator * allocator, gsize size,
return gst_msdk_video_memory_new (allocator);
}
static void
gst_msdk_video_allocator_free (GstAllocator * allocator, GstMemory * memory)
{
GstMsdkVideoAllocator *msdk_video_allocator =
GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
GstMsdkAllocResponse *resp =
gst_msdk_context_get_cached_alloc_responses
(msdk_video_allocator->context, msdk_video_allocator->alloc_response);
if (resp)
resp->num_used_memory--;
}
static void
gst_msdk_video_allocator_finalize (GObject * object)
{
......@@ -273,7 +277,6 @@ gst_msdk_video_allocator_class_init (GstMsdkVideoAllocatorClass * klass)
object_class->finalize = gst_msdk_video_allocator_finalize;
allocator_class->alloc = gst_msdk_video_allocator_alloc;
allocator_class->free = gst_msdk_video_allocator_free;
}
static void
......
......@@ -72,6 +72,12 @@ struct _GstMsdkVideoMemory
GstMemory *
gst_msdk_video_memory_new (GstAllocator * allocator);
gboolean
gst_msdk_video_memory_get_surface_available (GstMsdkVideoMemory * mem);
void
gst_msdk_video_memory_release_surface (GstMsdkVideoMemory * mem);
gboolean
gst_video_meta_map_msdk_memory (GstVideoMeta * meta, guint plane,
GstMapInfo * info, gpointer * data, gint * stride, GstMapFlags flags);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment