Commit d95a12e1 authored by Alessandro Decina's avatar Alessandro Decina
Browse files

vtdec: implement the GL texture upload meta

parent cdfe90aa
......@@ -26,12 +26,11 @@
#endif
#include "corevideotexturecache.h"
#include "coremediabuffer.h"
#include "corevideobuffer.h"
GstCoreVideoTextureCache *
gst_core_video_texture_cache_new (GstGLContext * ctx)
{
CVReturn err;
g_return_val_if_fail (ctx != NULL, NULL);
GstCoreVideoTextureCache *cache = g_new0 (GstCoreVideoTextureCache, 1);
......@@ -41,21 +40,21 @@ gst_core_video_texture_cache_new (GstGLContext * ctx)
CGLPixelFormatAttribute attribs[1] = { 0 };
int numPixelFormats;
CGLPixelFormatObj pixelFormat;
CGLChoosePixelFormat (attribs, &pixelFormat, &numPixelFormats); // 5
NSOpenGLContext *platform_ctx = (NSOpenGLContext *) gst_gl_context_get_gl_context (ctx);
err = CVOpenGLTextureCacheCreate (kCFAllocatorDefault, NULL,
[platform_ctx CGLContextObj], pixelFormat,
NULL, &cache->cache);
CGLChoosePixelFormat (attribs, &pixelFormat, &numPixelFormats); // 5
NSOpenGLContext *platform_ctx =
(NSOpenGLContext *) gst_gl_context_get_gl_context (ctx);
CVOpenGLTextureCacheCreate (kCFAllocatorDefault, NULL,
[platform_ctx CGLContextObj], pixelFormat, NULL, &cache->cache);
#else
err = CVOpenGLESTextureCacheCreate (kCFAllocatorDefault, NULL,
(CVEAGLContext) gst_gl_context_get_gl_context (ctx),
NULL, &cache->cache);
CVOpenGLESTextureCacheCreate (kCFAllocatorDefault, NULL,
(CVEAGLContext) gst_gl_context_get_gl_context (ctx), NULL, &cache->cache);
#endif
return cache;
}
void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
void
gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
{
g_return_if_fail (cache != NULL);
......@@ -69,11 +68,13 @@ void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
}
gboolean
gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4])
gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta,
guint texture_id[4])
{
g_return_val_if_fail (meta != NULL, FALSE);
GstCoreVideoTextureCache *cache = (GstCoreVideoTextureCache *) meta->user_data;
GstCoreVideoTextureCache *cache =
(GstCoreVideoTextureCache *) meta->user_data;
const GstGLFuncs *gl = cache->ctx->gl_vtable;
#if !HAVE_IOS
CVOpenGLTextureRef texture = NULL;
......@@ -81,16 +82,25 @@ gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint t
CVOpenGLESTextureRef texture = NULL;
#endif
GstVideoMeta *video_meta = gst_buffer_get_video_meta (meta->buffer);
GstCoreMediaMeta *cv_meta = (GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer,
gst_core_media_meta_api_get_type ());
GstCoreMediaMeta *cm_meta =
(GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer,
gst_core_media_meta_api_get_type ());
GstCoreVideoMeta *cv_meta =
(GstCoreVideoMeta *) gst_buffer_get_meta (meta->buffer,
gst_core_video_meta_api_get_type ());
CVPixelBufferRef pixel_buf;
if (cm_meta)
pixel_buf = cm_meta->pixel_buf;
else
pixel_buf = cv_meta->pixbuf;
#if !HAVE_IOS
CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
cache->cache, cv_meta->pixel_buf, NULL, &texture);
cache->cache, pixel_buf, NULL, &texture);
#else
CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
cache->cache, cv_meta->image_buf, NULL, GL_TEXTURE_2D,
GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE,
0, &texture);
cache->cache, cm_meta->image_buf, NULL, GL_TEXTURE_2D,
GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE,
0, &texture);
#endif
GLuint fboId;
gl->GenFramebuffers (1, &fboId);
......@@ -98,13 +108,14 @@ gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint t
gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
#if !HAVE_IOS
CVOpenGLTextureGetTarget(texture), CVOpenGLTextureGetName(texture),
CVOpenGLTextureGetTarget (texture), CVOpenGLTextureGetName (texture),
#else
CVOpenGLESTextureGetTarget(texture), CVOpenGLESTextureGetName(texture),
CVOpenGLESTextureGetTarget (texture), CVOpenGLESTextureGetName (texture),
#endif
0);
gl->BindTexture (GL_TEXTURE_2D, texture_id[0]);
gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width, video_meta->height, 0);
gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width,
video_meta->height, 0);
gl->BindTexture (GL_TEXTURE_2D, 0);
gl->BindFramebuffer (GL_FRAMEBUFFER, 0);
gl->DeleteFramebuffers (1, &fboId);
......
......@@ -35,13 +35,15 @@
#include "config.h"
#endif
#include <string.h>
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/video/gstvideodecoder.h>
#include <gst/gl/gstglcontext.h>
#include "vtdec.h"
#include "vtutil.h"
#include "corevideobuffer.h"
#include <string.h>
#include "coremediabuffer.h"
GST_DEBUG_CATEGORY_STATIC (gst_vtdec_debug_category);
#define GST_CAT_DEFAULT gst_vtdec_debug_category
......@@ -50,6 +52,8 @@ static void gst_vtdec_finalize (GObject * object);
static gboolean gst_vtdec_start (GstVideoDecoder * decoder);
static gboolean gst_vtdec_stop (GstVideoDecoder * decoder);
static gboolean gst_vtdec_decide_allocation (GstVideoDecoder * decoder,
GstQuery * query);
static gboolean gst_vtdec_set_format (GstVideoDecoder * decoder,
GstVideoCodecState * state);
static gboolean gst_vtdec_flush (GstVideoDecoder * decoder);
......@@ -57,7 +61,8 @@ static GstFlowReturn gst_vtdec_finish (GstVideoDecoder * decoder);
static GstFlowReturn gst_vtdec_handle_frame (GstVideoDecoder * decoder,
GstVideoCodecFrame * frame);
static gboolean gst_vtdec_create_session (GstVtdec * vtdec);
static gboolean gst_vtdec_create_session (GstVtdec * vtdec,
GstVideoFormat format);
static void gst_vtdec_invalidate_session (GstVtdec * vtdec);
static CMSampleBufferRef cm_sample_buffer_from_gst_buffer (GstVtdec * vtdec,
GstBuffer * buf);
......@@ -86,16 +91,6 @@ static GstStaticPadTemplate gst_vtdec_sink_template =
"video/mpeg, mpegversion=2;" "image/jpeg")
);
#ifdef HAVE_IOS
#define GST_VTDEC_VIDEO_FORMAT_STR "NV12"
#define GST_VTDEC_VIDEO_FORMAT GST_VIDEO_FORMAT_NV12
#define GST_VTDEC_CV_VIDEO_FORMAT kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
#else
#define GST_VTDEC_VIDEO_FORMAT_STR "UYVY"
#define GST_VTDEC_VIDEO_FORMAT GST_VIDEO_FORMAT_UYVY
#define GST_VTDEC_CV_VIDEO_FORMAT kCVPixelFormatType_422YpCbCr8
#endif
/* define EnableHardwareAcceleratedVideoDecoder in < 10.9 */
#if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED < 1090
const CFStringRef
......@@ -103,8 +98,17 @@ const CFStringRef
CFSTR ("EnableHardwareAcceleratedVideoDecoder");
#endif
#ifdef HAVE_IOS
#define GST_VTDEC_VIDEO_FORMAT_STR "NV12"
#else
#define GST_VTDEC_VIDEO_FORMAT_STR "UYVY"
#endif
#define VIDEO_SRC_CAPS \
GST_VIDEO_CAPS_MAKE("{" GST_VTDEC_VIDEO_FORMAT_STR "}")
GST_VIDEO_CAPS_MAKE(GST_VTDEC_VIDEO_FORMAT_STR) ";" \
GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
(GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, \
"RGBA") ";"
G_DEFINE_TYPE_WITH_CODE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER,
GST_DEBUG_CATEGORY_INIT (gst_vtdec_debug_category, "vtdec", 0,
......@@ -134,6 +138,8 @@ gst_vtdec_class_init (GstVtdecClass * klass)
gobject_class->finalize = gst_vtdec_finalize;
video_decoder_class->start = GST_DEBUG_FUNCPTR (gst_vtdec_start);
video_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_vtdec_stop);
video_decoder_class->decide_allocation =
GST_DEBUG_FUNCPTR (gst_vtdec_decide_allocation);
video_decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_vtdec_set_format);
video_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_vtdec_flush);
video_decoder_class->finish = GST_DEBUG_FUNCPTR (gst_vtdec_finish);
......@@ -156,6 +162,7 @@ gst_vtdec_finalize (GObject * object)
g_async_queue_unref (vtdec->reorder_queue);
G_OBJECT_CLASS (gst_vtdec_parent_class)->finalize (object);
}
......@@ -177,11 +184,64 @@ gst_vtdec_stop (GstVideoDecoder * decoder)
if (vtdec->session)
gst_vtdec_invalidate_session (vtdec);
if (vtdec->texture_cache)
gst_core_video_texture_cache_free (vtdec->texture_cache);
vtdec->texture_cache = NULL;
GST_DEBUG_OBJECT (vtdec, "stop");
return TRUE;
}
static gboolean
gst_vtdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
{
gboolean ret;
GstVtdec *vtdec = GST_VTDEC (decoder);
ret =
GST_VIDEO_DECODER_CLASS (gst_vtdec_parent_class)->decide_allocation
(decoder, query);
if (ret) {
guint idx;
if (gst_query_find_allocation_meta (query,
GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) {
GstGLContext *context;
const GstStructure *upload_meta_params;
gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params);
if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext",
GST_GL_TYPE_CONTEXT, &context, NULL) && context) {
vtdec->texture_cache = gst_core_video_texture_cache_new (context);
gst_object_unref (context);
}
}
}
return ret;
}
static GstVideoFormat
gst_vtdec_negotiate_output_format (GstVtdec * vtdec)
{
GstCaps *caps = NULL;
GstVideoFormat format;
GstStructure *structure;
const gchar *s;
caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec));
if (!caps)
caps = gst_pad_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), NULL);
caps = gst_caps_truncate (caps);
structure = gst_caps_get_structure (caps, 0);
s = gst_structure_get_string (structure, "format");
format = gst_video_format_from_string (s);
gst_caps_unref (caps);
return format;
}
static gboolean
gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
{
......@@ -190,6 +250,8 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
CMFormatDescriptionRef format_description = NULL;
const char *caps_name;
GstVtdec *vtdec = GST_VTDEC (decoder);
GstVideoFormat output_format;
GstVideoCodecState *output_state;
GST_DEBUG_OBJECT (vtdec, "set_format");
......@@ -229,12 +291,19 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
CFRelease (vtdec->format_description);
vtdec->format_description = format_description;
if (!gst_vtdec_create_session (vtdec))
output_format = gst_vtdec_negotiate_output_format (vtdec);
if (!gst_vtdec_create_session (vtdec, output_format))
return FALSE;
gst_video_decoder_set_output_state (decoder,
GST_VTDEC_VIDEO_FORMAT, vtdec->video_info.width, vtdec->video_info.height,
state);
output_state = gst_video_decoder_set_output_state (decoder,
output_format, vtdec->video_info.width, vtdec->video_info.height, state);
output_state->caps = gst_video_info_to_caps (&output_state->info);
if (output_state->info.finfo->format == GST_VIDEO_FORMAT_RGBA) {
gst_caps_set_features (output_state->caps, 0,
gst_caps_features_new
(GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, NULL));
}
return TRUE;
}
......@@ -312,12 +381,28 @@ error:
}
static gboolean
gst_vtdec_create_session (GstVtdec * vtdec)
gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format)
{
CFMutableDictionaryRef output_image_buffer_attrs;
VTDecompressionOutputCallbackRecord callback;
CFMutableDictionaryRef videoDecoderSpecification;
OSStatus status;
guint32 cv_format;
switch (format) {
case GST_VIDEO_FORMAT_NV12:
cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
break;
case GST_VIDEO_FORMAT_UYVY:
cv_format = kCVPixelFormatType_422YpCbCr8;
break;
case GST_VIDEO_FORMAT_RGBA:
cv_format = kCVPixelFormatType_32BGRA;
break;
default:
g_warn_if_reached ();
break;
}
videoDecoderSpecification =
CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
......@@ -333,7 +418,7 @@ gst_vtdec_create_session (GstVtdec * vtdec)
CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
&kCFTypeDictionaryValueCallBacks);
gst_vtutil_dict_set_i32 (output_image_buffer_attrs,
kCVPixelBufferPixelFormatTypeKey, GST_VTDEC_CV_VIDEO_FORMAT);
kCVPixelBufferPixelFormatTypeKey, cv_format);
gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferWidthKey,
vtdec->video_info.width);
gst_vtutil_dict_set_i32 (output_image_buffer_attrs, kCVPixelBufferHeightKey,
......@@ -557,13 +642,12 @@ gst_vtdec_session_output_callback (void *decompression_output_ref_con,
}
buf = gst_core_video_buffer_new (image_buffer, &state->info);
gst_video_codec_state_unref (state);
frame->output_buffer = buf;
gst_buffer_copy_into (buf, frame->input_buffer,
GST_BUFFER_COPY_METADATA | GST_BUFFER_COPY_FLAGS, 0, -1);
GST_BUFFER_PTS (buf) = pts.value;
GST_BUFFER_DURATION (buf) = duration.value;
frame->output_buffer = buf;
g_async_queue_push_sorted (vtdec->reorder_queue, frame,
sort_frames_by_pts, NULL);
......@@ -590,6 +674,11 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain,
GstFlowReturn ret = GST_FLOW_OK;
GstVideoDecoder *decoder = GST_VIDEO_DECODER (vtdec);
/* negotiate now so that we know whether we need to use the GL upload meta or
* not */
if (gst_pad_check_reconfigure (decoder->srcpad))
gst_video_decoder_negotiate (decoder);
if (drain)
VTDecompressionSessionWaitForAsynchronousFrames (vtdec->session);
......@@ -599,6 +688,15 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain,
while ((g_async_queue_length (vtdec->reorder_queue) >=
vtdec->reorder_queue_length) || drain || flush) {
frame = (GstVideoCodecFrame *) g_async_queue_try_pop (vtdec->reorder_queue);
if (vtdec->texture_cache != NULL) {
GstVideoGLTextureType texture_types[4] =
{ GST_VIDEO_GL_TEXTURE_TYPE_RGBA, 0 };
gst_buffer_add_video_gl_texture_upload_meta (frame->output_buffer,
GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL, 1, texture_types,
gst_core_video_texture_cache_upload, vtdec->texture_cache, NULL,
NULL);
}
/* we need to check this in case reorder_queue_length=0 (jpeg for
* example) or we're draining/flushing
*/
......
......@@ -25,6 +25,7 @@
#include <gst/video/gstvideodecoder.h>
#include <CoreMedia/CoreMedia.h>
#include <VideoToolbox/VideoToolbox.h>
#include "corevideotexturecache.h"
G_BEGIN_DECLS
......@@ -45,6 +46,7 @@ struct _GstVtdec
VTDecompressionSessionRef session;
GAsyncQueue *reorder_queue;
gint reorder_queue_length;
GstCoreVideoTextureCache *texture_cache;
};
struct _GstVtdecClass
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment