Commit d5c36937 authored by Matthew Waters's avatar Matthew Waters 🐨

gl: split glcolorconvert usage from glupload

the separation allows the transfer operation to occur in a separate
thread/time which may increase performance in specific circumstances.
parent 4ccf4582
......@@ -771,7 +771,7 @@ GST_GL_COLOR_CONVERT_VIDEO_CAPS
GstGLColorConvert
GstGLColorConvertClass
gst_gl_color_convert_new
gst_gl_color_convert_set_format
gst_gl_color_convert_set_caps
gst_gl_color_convert_perform
<SUBSECTION Standard>
GstGLColorConvertPrivate
......
......@@ -682,6 +682,11 @@ gst_glimage_sink_change_state (GstElement * element, GstStateChange transition)
glimage_sink->upload = NULL;
}
if (glimage_sink->convert) {
gst_object_unref (glimage_sink->convert);
glimage_sink->convert = NULL;
}
glimage_sink->window_id = 0;
//but do not reset glimage_sink->new_window_id
......@@ -760,6 +765,7 @@ gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
GstStructure *structure;
GstBufferPool *newpool, *oldpool;
GstCapsFeatures *gl_features;
GstCaps *uploaded_caps;
GST_DEBUG ("set caps with %" GST_PTR_FORMAT, caps);
......@@ -846,14 +852,30 @@ gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
gl_features =
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
uploaded_caps = gst_caps_copy (caps);
gst_caps_set_features (uploaded_caps, 0,
gst_caps_features_copy (gl_features));
gst_gl_upload_set_caps (glimage_sink->upload, caps, uploaded_caps);
if (glimage_sink->gl_caps)
gst_caps_unref (glimage_sink->gl_caps);
glimage_sink->gl_caps = gst_caps_copy (caps);
gst_caps_set_simple (glimage_sink->gl_caps, "format", G_TYPE_STRING, "RGBA",
NULL);
gst_caps_set_features (glimage_sink->gl_caps, 0, gl_features);
gst_caps_set_features (glimage_sink->gl_caps, 0,
gst_caps_features_copy (gl_features));
if (glimage_sink->convert)
gst_object_unref (glimage_sink->convert);
glimage_sink->convert = gst_gl_color_convert_new (glimage_sink->context);
if (!gst_gl_color_convert_set_caps (glimage_sink->convert, uploaded_caps,
glimage_sink->gl_caps)) {
gst_caps_features_free (gl_features);
return FALSE;
}
gst_caps_features_free (gl_features);
gst_gl_upload_set_caps (glimage_sink->upload, caps, glimage_sink->gl_caps);
glimage_sink->caps_change = TRUE;
return TRUE;
......@@ -863,7 +885,7 @@ static GstFlowReturn
gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
{
GstGLImageSink *glimage_sink;
GstBuffer *next_buffer = NULL;
GstBuffer *uploaded_buffer, *next_buffer = NULL;
GstVideoFrame gl_frame;
GstVideoInfo gl_info;
......@@ -879,17 +901,26 @@ gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
if (!_ensure_gl_setup (glimage_sink))
return GST_FLOW_NOT_NEGOTIATED;
if (!gst_gl_upload_perform_with_buffer (glimage_sink->upload, buf,
&next_buffer))
if (gst_gl_upload_perform_with_buffer (glimage_sink->upload, buf,
&uploaded_buffer) != GST_GL_UPLOAD_DONE)
goto upload_failed;
if (!(next_buffer =
gst_gl_color_convert_perform (glimage_sink->convert,
uploaded_buffer))) {
gst_buffer_unref (uploaded_buffer);
goto upload_failed;
}
gst_video_info_from_caps (&gl_info, glimage_sink->gl_caps);
if (!gst_video_frame_map (&gl_frame, &gl_info, next_buffer,
GST_MAP_READ | GST_MAP_GL)) {
gst_buffer_unref (uploaded_buffer);
gst_buffer_unref (next_buffer);
goto upload_failed;
}
gst_buffer_unref (uploaded_buffer);
glimage_sink->next_tex = *(guint *) gl_frame.data[0];
......
......@@ -69,6 +69,7 @@ struct _GstGLImageSink
gboolean handle_events;
GstGLUpload *upload;
GstGLColorConvert *convert;
guint next_tex;
GstBuffer *next_buffer;
......
......@@ -1104,7 +1104,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf)
walk = g_list_next (walk);
if (vaggpad->buffer != NULL) {
GstBuffer *gl_buf;
GstBuffer *uploaded_buf;
GstCaps *gl_caps;
GstCapsFeatures *gl_features;
GstVideoInfo gl_info;
......@@ -1119,45 +1119,68 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf)
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
gl_caps = gst_video_info_to_caps (&gl_info);
gst_caps_set_features (gl_caps, 0, gl_features);
gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features));
if (!pad->upload) {
GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad));
GstCaps *upload_caps = gst_caps_copy (in_caps);
pad->upload = gst_gl_upload_new (mix->context);
gst_gl_upload_set_caps (pad->upload, in_caps, gl_caps);
gst_caps_set_features (upload_caps, 0,
gst_caps_features_copy (gl_features));
gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps);
if (!pad->convert) {
pad->convert = gst_gl_color_convert_new (mix->context);
gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps);
}
gst_caps_unref (upload_caps);
gst_caps_unref (in_caps);
}
gst_caps_features_free (gl_features);
gst_caps_unref (gl_caps);
sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer);
if (sync_meta)
gst_gl_sync_meta_wait (sync_meta);
if (!gst_gl_upload_perform_with_buffer (pad->upload,
vaggpad->buffer, &gl_buf)) {
if (gst_gl_upload_perform_with_buffer (pad->upload,
vaggpad->buffer, &uploaded_buf) != GST_GL_UPLOAD_DONE) {
++array_index;
pad->mapped = FALSE;
gst_caps_unref (gl_caps);
continue;
}
if (!gst_video_frame_map (&gl_frame, &gl_info, gl_buf,
if (pad->gl_buffer)
gst_buffer_unref (pad->gl_buffer);
if (!(pad->gl_buffer =
gst_gl_color_convert_perform (pad->convert, uploaded_buf))) {
++array_index;
pad->mapped = FALSE;
gst_buffer_unref (uploaded_buf);
continue;
}
if (!gst_video_frame_map (&gl_frame, &gl_info, pad->gl_buffer,
GST_MAP_READ | GST_MAP_GL)) {
++array_index;
pad->mapped = FALSE;
gst_buffer_unref (gl_buf);
gst_caps_unref (gl_caps);
gst_buffer_unref (uploaded_buf);
gst_buffer_unref (pad->gl_buffer);
pad->gl_buffer = NULL;
continue;
}
pad->mapped = TRUE;
frame->texture = *(guint *) gl_frame.data[0];
gst_caps_unref (gl_caps);
gst_buffer_unref (uploaded_buf);
gst_video_frame_unmap (&gl_frame);
gst_buffer_unref (gl_buf);
}
++array_index;
}
......@@ -1292,11 +1315,21 @@ _clean_upload (GstAggregator * agg, GstAggregatorPad * aggpad, gpointer udata)
{
GstGLMixerPad *pad = GST_GL_MIXER_PAD (aggpad);
if (pad->gl_buffer) {
gst_buffer_unref (pad->gl_buffer);
pad->gl_buffer = NULL;
}
if (pad->upload) {
gst_object_unref (pad->upload);
pad->upload = NULL;
}
if (pad->convert) {
gst_object_unref (pad->convert);
pad->convert = NULL;
}
return TRUE;
}
......
......@@ -49,7 +49,8 @@ struct _GstGLMixerPad
/* <private> */
GstGLUpload *upload;
guint in_tex_id;
GstGLColorConvert *convert;
GstBuffer *gl_buffer;
gboolean mapped;
};
......
......@@ -545,47 +545,76 @@ gst_gl_color_convert_reset (GstGLColorConvert * convert)
}
}
static void
_gst_gl_color_convert_set_format_unlocked (GstGLColorConvert * convert,
GstVideoInfo * in_info, GstVideoInfo * out_info)
static gboolean
_gst_gl_color_convert_set_caps_unlocked (GstGLColorConvert * convert,
GstCaps * in_caps, GstCaps * out_caps)
{
g_return_if_fail (convert != NULL);
g_return_if_fail (in_info);
g_return_if_fail (out_info);
g_return_if_fail (GST_VIDEO_INFO_FORMAT (in_info) !=
GST_VIDEO_FORMAT_UNKNOWN);
g_return_if_fail (GST_VIDEO_INFO_FORMAT (in_info) !=
GST_VIDEO_FORMAT_ENCODED);
g_return_if_fail (GST_VIDEO_INFO_FORMAT (out_info) !=
GST_VIDEO_FORMAT_UNKNOWN);
g_return_if_fail (GST_VIDEO_INFO_FORMAT (out_info) !=
GST_VIDEO_FORMAT_ENCODED);
if (gst_video_info_is_equal (&convert->in_info, in_info) &&
gst_video_info_is_equal (&convert->out_info, out_info))
return;
GstVideoInfo in_info, out_info;
GstCapsFeatures *in_features, *out_features, *gl_features;
g_return_val_if_fail (convert != NULL, FALSE);
g_return_val_if_fail (in_caps, FALSE);
g_return_val_if_fail (out_caps, FALSE);
if (!gst_video_info_from_caps (&in_info, in_caps))
g_assert_not_reached ();
if (!gst_video_info_from_caps (&out_info, out_caps))
g_assert_not_reached ();
g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&in_info) !=
GST_VIDEO_FORMAT_UNKNOWN, FALSE);
g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&in_info) !=
GST_VIDEO_FORMAT_ENCODED, FALSE);
g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&out_info) !=
GST_VIDEO_FORMAT_UNKNOWN, FALSE);
g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&out_info) !=
GST_VIDEO_FORMAT_ENCODED, FALSE);
gl_features =
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
in_features = gst_caps_get_features (in_caps, 0);
out_features = gst_caps_get_features (out_caps, 0);
if (!gst_caps_features_is_equal (in_features, gl_features)
|| !gst_caps_features_is_equal (out_features, gl_features)) {
gst_caps_features_free (gl_features);
return FALSE;
}
gst_caps_features_free (gl_features);
if (gst_video_info_is_equal (&convert->in_info, &in_info) &&
gst_video_info_is_equal (&convert->out_info, &out_info))
return TRUE;
gst_gl_color_convert_reset (convert);
convert->in_info = *in_info;
convert->out_info = *out_info;
convert->in_info = in_info;
convert->out_info = out_info;
convert->initted = FALSE;
return TRUE;
}
/**
* gst_gl_color_convert_set_format:
* gst_gl_color_convert_set_caps:
* @convert: a #GstGLColorConvert
* @in_info: input #GstVideoInfo
* @out_info: output #GstVideoInfo
*
* Initializes @convert with the information required for conversion.
*/
void
gst_gl_color_convert_set_format (GstGLColorConvert * convert,
GstVideoInfo * in_info, GstVideoInfo * out_info)
gboolean
gst_gl_color_convert_set_caps (GstGLColorConvert * convert,
GstCaps * in_caps, GstCaps * out_caps)
{
gboolean ret;
GST_OBJECT_LOCK (convert);
_gst_gl_color_convert_set_format_unlocked (convert, in_info, out_info);
ret = _gst_gl_color_convert_set_caps_unlocked (convert, in_caps, out_caps);
GST_OBJECT_UNLOCK (convert);
return ret;
}
/**
......@@ -594,7 +623,7 @@ gst_gl_color_convert_set_format (GstGLColorConvert * convert,
* @inbuf: the texture ids for input formatted according to in_info
*
* Converts the data contained by @inbuf using the formats specified by the
* #GstVideoInfo<!-- -->s passed to gst_gl_color_convert_set_format()
* #GstVideoInfo<!-- -->s passed to gst_gl_color_convert_set_caps()
*
* Returns: a converted #GstBuffer or %NULL%
*/
......
......@@ -97,9 +97,9 @@ struct _GstGLColorConvertClass
GstGLColorConvert * gst_gl_color_convert_new (GstGLContext * context);
void gst_gl_color_convert_set_format (GstGLColorConvert * convert,
GstVideoInfo * in_info,
GstVideoInfo * out_info);
gboolean gst_gl_color_convert_set_caps (GstGLColorConvert * convert,
GstCaps * in_caps,
GstCaps * out_caps);
GstBuffer * gst_gl_color_convert_perform (GstGLColorConvert * convert, GstBuffer * inbuf);
......
......@@ -246,6 +246,9 @@ _init_download (GstGLDownload * download)
GstVideoFormat v_format;
guint out_width, out_height;
GstVideoInfo in_info;
GstCaps *in_caps, *out_caps;
GstCapsFeatures *in_gl_features, *out_gl_features;
gboolean res;
v_format = GST_VIDEO_INFO_FORMAT (&download->info);
out_width = GST_VIDEO_INFO_WIDTH (&download->info);
......@@ -266,13 +269,24 @@ _init_download (GstGLDownload * download)
}
}
in_gl_features =
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
gst_video_info_set_format (&in_info, GST_VIDEO_FORMAT_RGBA, out_width,
out_height);
in_caps = gst_video_info_to_caps (&in_info);
gst_caps_set_features (in_caps, 0, in_gl_features);
gst_gl_color_convert_set_format (download->convert, &in_info,
&download->info);
out_gl_features =
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
out_caps = gst_video_info_to_caps (&download->info);
gst_caps_set_features (out_caps, 0, out_gl_features);
return TRUE;
res = gst_gl_color_convert_set_caps (download->convert, in_caps, out_caps);
gst_caps_unref (in_caps);
gst_caps_unref (out_caps);
return res;
}
static gboolean
......
......@@ -253,6 +253,11 @@ gst_gl_filter_reset (GstGLFilter * filter)
filter->upload = NULL;
}
if (filter->in_convert) {
gst_object_unref (filter->in_convert);
filter->in_convert = NULL;
}
if (filter->download) {
gst_object_unref (filter->download);
filter->download = NULL;
......@@ -309,9 +314,9 @@ gst_gl_filter_reset (GstGLFilter * filter)
gst_object_unref (filter->context);
filter->context = NULL;
if (filter->in_gl_caps) {
gst_caps_unref (filter->in_gl_caps);
filter->in_gl_caps = NULL;
if (filter->in_converted_caps) {
gst_caps_unref (filter->in_converted_caps);
filter->in_converted_caps = NULL;
}
}
......@@ -1225,26 +1230,34 @@ gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf,
gst_caps_features_is_equal (GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY,
gst_caps_get_features (filter->out_caps, 0));
GstMapFlags out_map_flags = GST_MAP_WRITE;
GstBuffer *upload_buffer;
filter_class = GST_GL_FILTER_GET_CLASS (filter);
if (gst_gl_upload_perform_with_buffer (filter->upload, inbuf,
&upload_buffer) != GST_GL_UPLOAD_DONE) {
return FALSE;
}
if (filter->uploaded_buffer) {
gst_buffer_unref (filter->uploaded_buffer);
filter->uploaded_buffer = NULL;
}
if (!gst_gl_upload_perform_with_buffer (filter->upload, inbuf,
&filter->uploaded_buffer)) {
filter->uploaded_buffer =
gst_gl_color_convert_perform (filter->in_convert, upload_buffer);
if (!filter->uploaded_buffer) {
ret = FALSE;
goto upload_error;
goto inbuf_error;
}
gst_video_info_from_caps (&gl_info, filter->in_gl_caps);
gst_video_info_from_caps (&gl_info, filter->in_converted_caps);
if (!gst_video_frame_map (&gl_frame, &gl_info, filter->uploaded_buffer,
GST_MAP_READ | GST_MAP_GL)) {
ret = FALSE;
goto upload_error;
goto inbuf_error;
}
in_tex = *(guint *) gl_frame.data[0];
......@@ -1257,7 +1270,7 @@ gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf,
if (!gst_video_frame_map (&out_frame, &filter->out_info, outbuf,
out_map_flags)) {
ret = FALSE;
goto inbuf_error;
goto unmap_out_error;
}
if (!to_download) {
......@@ -1290,10 +1303,11 @@ gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf,
}
gst_video_frame_unmap (&gl_frame);
upload_error:
unmap_out_error:
gst_video_frame_unmap (&out_frame);
inbuf_error:
gst_gl_upload_release_buffer (filter->upload);
gst_buffer_unref (upload_buffer);
return ret;
}
......@@ -1315,6 +1329,7 @@ gst_gl_filter_transform (GstBaseTransform * bt, GstBuffer * inbuf,
if (!filter->upload) {
GstCaps *in_caps =
gst_pad_get_current_caps (GST_BASE_TRANSFORM_SINK_PAD (bt));
GstCaps *uploaded_caps;
GstCapsFeatures *out_features;
GstVideoInfo out_info;
......@@ -1325,16 +1340,42 @@ gst_gl_filter_transform (GstBaseTransform * bt, GstBuffer * inbuf,
out_features =
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
if (filter->in_gl_caps)
gst_caps_unref (filter->in_gl_caps);
filter->in_gl_caps = gst_video_info_to_caps (&out_info);
gst_caps_set_features (filter->in_gl_caps, 0, out_features);
uploaded_caps = gst_caps_copy (in_caps);
gst_caps_set_features (uploaded_caps, 0, out_features);
filter->upload = gst_gl_upload_new (filter->context);
if (!gst_gl_upload_set_caps (filter->upload, in_caps, filter->in_gl_caps))
if (!gst_gl_upload_set_caps (filter->upload, in_caps, uploaded_caps)) {
gst_caps_unref (in_caps);
return GST_FLOW_ERROR;
}
gst_caps_unref (in_caps);
if (!filter->in_convert) {
GstCapsFeatures *converted_features;
GstVideoInfo converted_info;
gst_video_info_set_format (&converted_info,
GST_VIDEO_FORMAT_RGBA,
GST_VIDEO_INFO_WIDTH (&filter->in_info),
GST_VIDEO_INFO_HEIGHT (&filter->in_info));
converted_features =
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
if (filter->in_converted_caps)
gst_caps_unref (filter->in_converted_caps);
filter->in_converted_caps = gst_video_info_to_caps (&converted_info);
gst_caps_set_features (filter->in_converted_caps, 0, converted_features);
filter->in_convert = gst_gl_color_convert_new (filter->context);
if (!gst_gl_color_convert_set_caps (filter->in_convert, uploaded_caps,
filter->in_converted_caps)) {
gst_caps_unref (uploaded_caps);
return GST_FLOW_ERROR;
}
}
gst_caps_unref (uploaded_caps);
}
g_assert (filter_class->filter || filter_class->filter_texture);
......
......@@ -75,6 +75,9 @@ struct _GstGLFilter
GLuint depthbuffer;
GstGLUpload *upload;
GstGLColorConvert *in_convert;
GstCaps *in_converted_caps;
GstGLDownload *download;
GstBuffer *uploaded_buffer;
......@@ -88,8 +91,6 @@ struct _GstGLFilter
GstGLContext *context;
GstGLContext *other_context;
GstCaps *in_gl_caps;
GLuint vao;
GLuint vertex_buffer;
GLint draw_attr_position_loc;
......
......@@ -68,16 +68,6 @@ struct _GstGLUploadPrivate
int method_i;
};
typedef enum
{
GST_GL_UPLOAD_DONE = 1,
GST_GL_UPLOAD_CONVERT_GL_MEMORY = 2,
GST_GL_UPLOAD_ERROR = -1,
GST_GL_UPLOAD_UNSUPPORTED = -2,
GST_GL_UPLOAD_UNSHARED_GL_CONTEXT = 3,
} GstGLUploadReturn;
typedef enum
{
METHOD_FLAG_CAN_SHARE_CONTEXT = 1,
......@@ -165,7 +155,7 @@ _gl_memory_upload_perform (gpointer impl, GstBuffer * buffer,
*outbuf = gst_buffer_ref (buffer);
return GST_GL_UPLOAD_CONVERT_GL_MEMORY;
return GST_GL_UPLOAD_DONE;
}
static void
......@@ -277,7 +267,7 @@ _egl_image_upload_perform (gpointer impl, GstBuffer * buffer,
gst_gl_buffer_pool_replace_last_buffer (GST_GL_BUFFER_POOL (buffer->pool),
buffer);
return GST_GL_UPLOAD_CONVERT_GL_MEMORY;
return GST_GL_UPLOAD_DONE;
}
static void
......@@ -421,7 +411,7 @@ _upload_meta_upload_perform (gpointer impl, GstBuffer * buffer,
if (!upload->result)
return GST_GL_UPLOAD_ERROR;
return GST_GL_UPLOAD_CONVERT_GL_MEMORY;
return GST_GL_UPLOAD_DONE;
}
static void
......@@ -522,7 +512,7 @@ _raw_data_upload_perform (gpointer impl, GstBuffer * buffer,
gst_memory_ref ((GstMemory *) raw->in_tex[i]));
}
return GST_GL_UPLOAD_CONVERT_GL_MEMORY;
return GST_GL_UPLOAD_DONE;
}
static void
......@@ -768,11 +758,11 @@ _upload_find_method (GstGLUpload * upload)
*
* Returns: whether the upload was successful
*/
gboolean
GstGLUploadReturn
gst_gl_upload_perform_with_buffer (GstGLUpload * upload, GstBuffer * buffer,
GstBuffer ** outbuf_ptr)
{
GstGLUploadReturn ret;
GstGLUploadReturn ret = GST_GL_UPLOAD_ERROR;
g_return_val_if_fail (GST_IS_GL_UPLOAD (upload), FALSE);
g_return_val_if_fail (GST_IS_BUFFER (buffer), FALSE);
......@@ -806,22 +796,6 @@ restart:
upload->priv->method = &_raw_data_upload;
upload->priv->method_impl = upload->priv->method->new (upload);
goto restart;
} else if (ret == GST_GL_UPLOAD_CONVERT_GL_MEMORY) {
GstBuffer *outbuf;
if (!upload->priv->convert)
upload->priv->convert = gst_gl_color_convert_new (upload->context);
gst_gl_color_convert_set_format (upload->priv->convert,
&upload->priv->in_info, &upload->priv->out_info);
if (!(outbuf = gst_gl_color_convert_perform (upload->priv->convert,
upload->priv->outbuf))) {
gst_gl_upload_release_buffer_unlocked (upload);
NEXT_METHOD;
}
gst_buffer_unref (upload->priv->outbuf);
upload->priv->outbuf = outbuf;
} else if (ret == GST_GL_UPLOAD_DONE) {
/* we are done */
} else {
......@@ -835,7 +809,7 @@ restart: