gstglimagesink.c 51.6 KB
Newer Older
1 2 3 4 5
/*
 * GStreamer
 * Copyright (C) 2003 Julien Moutte <julien@moutte.net>
 * Copyright (C) 2005,2006,2007 David A. Schleef <ds@schleef.org>
 * Copyright (C) 2008 Julien Isorce <julien.isorce@gmail.com>
6
 * Copyright (C) 2015 Matthew Waters <matthew@centricular.com>
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/**
 * SECTION:element-glimagesink
 *
 * glimagesink renders video frames to a drawable on a local or remote
 * display using OpenGL. This element can receive a Window ID from the
 * application through the VideoOverlay interface and will then render video
 * frames in this drawable.
 * If no Window ID was provided by the application, the element will
 * create its own internal window and render into it.
 *
34 35 36
 * See the #GstGLDisplay documentation for a list of environment variables that
 * can override window/platform detection.
 *
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
 * <refsect2>
 * <title>Scaling</title>
 * <para>
 * Depends on the driver, OpenGL handles hardware accelerated
 * scaling of video frames. This means that the element will just accept
 * incoming video frames no matter their geometry and will then put them to the
 * drawable scaling them on the fly. Using the #GstGLImageSink:force-aspect-ratio
 * property it is possible to enforce scaling with a constant aspect ratio,
 * which means drawing black borders around the video frame.
 * </para>
 * </refsect2>
 * <refsect2>
 * <title>Events</title>
 * <para>
 * Through the gl thread, glimagesink handle some events coming from the drawable
 * to manage its appearance even when the data is not flowing (GST_STATE_PAUSED).
 * That means that even when the element is paused, it will receive expose events
 * from the drawable and draw the latest frame with correct borders/aspect-ratio.
 * </para>
 * </refsect2>
 * <refsect2>
 * <title>Examples</title>
 * |[
60
 * gst-launch-1.0 -v videotestsrc ! video/x-raw ! glimagesink
61 62 63 64
 * ]| A pipeline to test hardware scaling.
 * No special opengl extension is used in this pipeline, that's why it should work
 * with OpenGL >= 1.1. That's the case if you are using the MESA3D driver v1.3.
 * |[
65
 * gst-launch-1.0 -v videotestsrc ! video/x-raw,format=I420 ! glimagesink
66 67 68 69 70 71 72 73
 * ]| A pipeline to test hardware scaling and hardware colorspace conversion.
 * When your driver supports GLSL (OpenGL Shading Language needs OpenGL >= 2.1),
 * the 4 following format YUY2, UYVY, I420, YV12 and AYUV are converted to RGB32
 * through some fragment shaders and using one framebuffer (FBO extension OpenGL >= 1.4).
 * If your driver does not support GLSL but supports MESA_YCbCr extension then
 * the you can use YUY2 and UYVY. In this case the colorspace conversion is automatically
 * made when loading the texture and therefore no framebuffer is used.
 * |[
74
 * gst-launch-1.0 -v gltestsrc ! glimagesink
75 76 77 78
 * ]| A pipeline 100% OpenGL.
 * No special opengl extension is used in this pipeline, that's why it should work
 * with OpenGL >= 1.1. That's the case if you are using the MESA3D driver v1.3.
 * |[
79
 * gst-plugins-bas/tests/examples/gl/generic/cube
80 81 82 83 84 85 86 87 88 89 90
 * ]| The graphic FPS scene can be greater than the input video FPS.
 * The graphic scene can be written from a client code through the
 * two glfilterapp properties.
 * </refsect2>
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <gst/video/videooverlay.h>
91
#include <gst/video/navigation.h>
92 93

#include "gstglimagesink.h"
94
#include "gstglsinkbin.h"
95

96 97 98 99
#if GST_GL_HAVE_PLATFORM_EGL
#include <gst/gl/egl/gsteglimagememory.h>
#endif

100 101 102
GST_DEBUG_CATEGORY (gst_debug_glimage_sink);
#define GST_CAT_DEFAULT gst_debug_glimage_sink

103 104 105 106 107 108 109 110 111 112 113 114 115 116
#define DEFAULT_SHOW_PREROLL_FRAME  TRUE
#define DEFAULT_SYNC                TRUE
#define DEFAULT_MAX_LATENESS        -1
#define DEFAULT_QOS                 FALSE
#define DEFAULT_ASYNC               TRUE
#define DEFAULT_TS_OFFSET           0
#define DEFAULT_BLOCKSIZE           4096
#define DEFAULT_RENDER_DELAY        0
#define DEFAULT_ENABLE_LAST_SAMPLE  TRUE
#define DEFAULT_THROTTLE_TIME       0
#define DEFAULT_MAX_BITRATE         0
#define DEFAULT_HANDLE_EVENTS       TRUE
#define DEFAULT_FORCE_ASPECT_RATIO  TRUE

117 118 119 120 121 122 123 124 125
typedef GstGLSinkBin GstGLImageSinkBin;
typedef GstGLSinkBinClass GstGLImageSinkBinClass;

G_DEFINE_TYPE (GstGLImageSinkBin, gst_gl_image_sink_bin, GST_TYPE_GL_SINK_BIN);

enum
{
  PROP_BIN_0,
  PROP_BIN_FORCE_ASPECT_RATIO,
126
  PROP_BIN_PIXEL_ASPECT_RATIO,
127 128 129 130 131 132 133 134 135
  PROP_BIN_HANDLE_EVENTS,
  PROP_BIN_CONTEXT,
  PROP_BIN_SHOW_PREROLL_FRAME,
  PROP_BIN_SYNC,
  PROP_BIN_MAX_LATENESS,
  PROP_BIN_QOS,
  PROP_BIN_ASYNC,
  PROP_BIN_TS_OFFSET,
  PROP_BIN_ENABLE_LAST_SAMPLE,
136
  PROP_BIN_LAST_SAMPLE,
137 138 139 140
  PROP_BIN_BLOCKSIZE,
  PROP_BIN_RENDER_DELAY,
  PROP_BIN_THROTTLE_TIME,
  PROP_BIN_MAX_BITRATE,
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
};

enum
{
  SIGNAL_BIN_0,
  SIGNAL_BIN_CLIENT_DRAW,
  SIGNAL_BIN_CLIENT_RESHAPE,
  SIGNAL_BIN_LAST,
};

static guint gst_gl_image_sink_bin_signals[SIGNAL_BIN_LAST] = { 0 };

static void
gst_gl_image_sink_bin_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * param_spec)
{
  g_object_set_property (G_OBJECT (GST_GL_SINK_BIN (object)->sink),
      param_spec->name, value);
}

static void
gst_gl_image_sink_bin_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * param_spec)
{
  g_object_get_property (G_OBJECT (GST_GL_SINK_BIN (object)->sink),
      param_spec->name, value);
}

static gboolean
_on_client_reshape (GstGLImageSink * sink, GstGLContext * context,
    guint width, guint height, gpointer data)
{
  gboolean ret;

  g_signal_emit (data, gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_RESHAPE],
      0, context, width, height, &ret);

  return ret;
}

static gboolean
_on_client_draw (GstGLImageSink * sink, GstGLContext * context,
183
    GstSample * sample, gpointer data)
184 185 186 187
{
  gboolean ret;

  g_signal_emit (data, gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_DRAW], 0,
188
      context, sample, &ret);
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213

  return ret;
}

static void
gst_gl_image_sink_bin_init (GstGLImageSinkBin * self)
{
  GstGLImageSink *sink = g_object_new (GST_TYPE_GLIMAGE_SINK, NULL);

  g_signal_connect (sink, "client-reshape", (GCallback) _on_client_reshape,
      self);
  g_signal_connect (sink, "client-draw", (GCallback) _on_client_draw, self);

  gst_gl_sink_bin_finish_init_with_element (GST_GL_SINK_BIN (self),
      GST_ELEMENT (sink));
}

static void
gst_gl_image_sink_bin_class_init (GstGLImageSinkBinClass * klass)
{
  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);

  gobject_class->get_property = gst_gl_image_sink_bin_get_property;
  gobject_class->set_property = gst_gl_image_sink_bin_set_property;

214
  /* gl sink */
215 216 217
  g_object_class_install_property (gobject_class, PROP_BIN_FORCE_ASPECT_RATIO,
      g_param_spec_boolean ("force-aspect-ratio",
          "Force aspect ratio",
218 219
          "When enabled, scaling will respect original aspect ratio",
          DEFAULT_FORCE_ASPECT_RATIO,
220
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
221 222 223 224 225 226 227
  g_object_class_install_property (gobject_class, PROP_BIN_HANDLE_EVENTS,
      g_param_spec_boolean ("ignore-alpha", "Ignore Alpha",
          "When enabled, alpha will be ignored and converted to black",
          DEFAULT_HANDLE_EVENTS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_BIN_CONTEXT,
      g_param_spec_object ("context", "OpenGL context", "Get OpenGL context",
          GST_GL_TYPE_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
228 229 230 231
  g_object_class_install_property (gobject_class, PROP_BIN_PIXEL_ASPECT_RATIO,
      gst_param_spec_fraction ("pixel-aspect-ratio", "Pixel Aspect Ratio",
          "The pixel aspect ratio of the device", 0, 1, G_MAXINT, 1, 1, 1,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
232

233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
  /* video sink */
  g_object_class_install_property (gobject_class, PROP_BIN_SHOW_PREROLL_FRAME,
      g_param_spec_boolean ("show-preroll-frame", "Show preroll frame",
          "Whether to render video frames during preroll",
          DEFAULT_SHOW_PREROLL_FRAME,
          G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));

  /* base sink */
  g_object_class_install_property (gobject_class, PROP_BIN_SYNC,
      g_param_spec_boolean ("sync", "Sync", "Sync on the clock", DEFAULT_SYNC,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_BIN_MAX_LATENESS,
      g_param_spec_int64 ("max-lateness", "Max Lateness",
          "Maximum number of nanoseconds that a buffer can be late before it "
          "is dropped (-1 unlimited)", -1, G_MAXINT64, DEFAULT_MAX_LATENESS,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_BIN_QOS,
      g_param_spec_boolean ("qos", "Qos",
          "Generate Quality-of-Service events upstream", DEFAULT_QOS,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_BIN_ASYNC,
      g_param_spec_boolean ("async", "Async",
          "Go asynchronously to PAUSED", DEFAULT_ASYNC,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_BIN_TS_OFFSET,
      g_param_spec_int64 ("ts-offset", "TS Offset",
          "Timestamp offset in nanoseconds", G_MININT64, G_MAXINT64,
          DEFAULT_TS_OFFSET, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_BIN_ENABLE_LAST_SAMPLE,
      g_param_spec_boolean ("enable-last-sample", "Enable Last Buffer",
          "Enable the last-sample property", DEFAULT_ENABLE_LAST_SAMPLE,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
265 266 267 268
  g_object_class_install_property (gobject_class, PROP_BIN_LAST_SAMPLE,
      g_param_spec_boxed ("last-sample", "Last Sample",
          "The last sample received in the sink", GST_TYPE_SAMPLE,
          G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
  g_object_class_install_property (gobject_class, PROP_BIN_BLOCKSIZE,
      g_param_spec_uint ("blocksize", "Block size",
          "Size in bytes to pull per buffer (0 = default)", 0, G_MAXUINT,
          DEFAULT_BLOCKSIZE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_BIN_RENDER_DELAY,
      g_param_spec_uint64 ("render-delay", "Render Delay",
          "Additional render delay of the sink in nanoseconds", 0, G_MAXUINT64,
          DEFAULT_RENDER_DELAY, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_BIN_THROTTLE_TIME,
      g_param_spec_uint64 ("throttle-time", "Throttle time",
          "The time to keep between rendered buffers (0 = disabled)", 0,
          G_MAXUINT64, DEFAULT_THROTTLE_TIME,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_BIN_MAX_BITRATE,
      g_param_spec_uint64 ("max-bitrate", "Max Bitrate",
          "The maximum bits per second to render (0 = disabled)", 0,
          G_MAXUINT64, DEFAULT_MAX_BITRATE,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
287 288 289 290

  gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_DRAW] =
      g_signal_new ("client-draw", G_TYPE_FROM_CLASS (klass),
      G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
291
      G_TYPE_BOOLEAN, 2, GST_GL_TYPE_CONTEXT, GST_TYPE_SAMPLE);
292 293 294 295 296 297 298

  gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_RESHAPE] =
      g_signal_new ("client-reshape", G_TYPE_FROM_CLASS (klass),
      G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
      G_TYPE_BOOLEAN, 3, GST_GL_TYPE_CONTEXT, G_TYPE_UINT, G_TYPE_UINT);
}

299 300 301 302 303 304 305
#define GST_GLIMAGE_SINK_GET_LOCK(glsink) \
  (GST_GLIMAGE_SINK(glsink)->drawing_lock)
#define GST_GLIMAGE_SINK_LOCK(glsink) \
  (g_mutex_lock(&GST_GLIMAGE_SINK_GET_LOCK (glsink)))
#define GST_GLIMAGE_SINK_UNLOCK(glsink) \
  (g_mutex_unlock(&GST_GLIMAGE_SINK_GET_LOCK (glsink)))

306 307 308 309 310
#define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0))
#define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1))
#define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0))
#define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
#define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
311

312 313
#define SUPPORTED_GL_APIS GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3

314
static void gst_glimage_sink_thread_init_redisplay (GstGLImageSink * gl_sink);
315
static void gst_glimage_sink_cleanup_glthread (GstGLImageSink * gl_sink);
316
static void gst_glimage_sink_on_close (GstGLImageSink * gl_sink);
317
static void gst_glimage_sink_on_resize (GstGLImageSink * gl_sink,
318
    gint width, gint height);
319
static void gst_glimage_sink_on_draw (GstGLImageSink * gl_sink);
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
static gboolean gst_glimage_sink_redisplay (GstGLImageSink * gl_sink);

static void gst_glimage_sink_finalize (GObject * object);
static void gst_glimage_sink_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * param_spec);
static void gst_glimage_sink_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * param_spec);

static gboolean gst_glimage_sink_stop (GstBaseSink * bsink);

static gboolean gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query);
static void gst_glimage_sink_set_context (GstElement * element,
    GstContext * context);

static GstStateChangeReturn
gst_glimage_sink_change_state (GstElement * element, GstStateChange transition);

static void gst_glimage_sink_get_times (GstBaseSink * bsink, GstBuffer * buf,
    GstClockTime * start, GstClockTime * end);
static gboolean gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps);
340 341
static GstCaps *gst_glimage_sink_get_caps (GstBaseSink * bsink,
    GstCaps * filter);
342 343
static GstFlowReturn gst_glimage_sink_prepare (GstBaseSink * bsink,
    GstBuffer * buf);
344
static GstFlowReturn gst_glimage_sink_show_frame (GstVideoSink * bsink,
345 346 347 348 349 350 351 352 353 354
    GstBuffer * buf);
static gboolean gst_glimage_sink_propose_allocation (GstBaseSink * bsink,
    GstQuery * query);

static void gst_glimage_sink_video_overlay_init (GstVideoOverlayInterface *
    iface);
static void gst_glimage_sink_set_window_handle (GstVideoOverlay * overlay,
    guintptr id);
static void gst_glimage_sink_expose (GstVideoOverlay * overlay);

355 356 357 358
static void
gst_glimage_sink_handle_events (GstVideoOverlay * overlay,
    gboolean handle_events);

359
static GstStaticPadTemplate gst_glimage_sink_template =
360
GST_STATIC_PAD_TEMPLATE ("sink",
361 362
    GST_PAD_SINK,
    GST_PAD_ALWAYS,
363 364
    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
        (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
365
            "RGBA"))
366 367 368 369 370 371 372 373
    );

enum
{
  ARG_0,
  ARG_DISPLAY,
  PROP_FORCE_ASPECT_RATIO,
  PROP_PIXEL_ASPECT_RATIO,
374
  PROP_CONTEXT,
375 376
  PROP_HANDLE_EVENTS,
  PROP_IGNORE_ALPHA,
377 378
};

379 380 381 382 383 384 385 386 387 388
enum
{
  SIGNAL_0,
  CLIENT_DRAW_SIGNAL,
  CLIENT_RESHAPE_SIGNAL,
  LAST_SIGNAL
};

static guint gst_glimage_sink_signals[LAST_SIGNAL] = { 0 };

389 390 391 392 393 394 395
static void
gst_glimage_sink_navigation_send_event (GstNavigation * navigation, GstStructure
    * structure)
{
  GstGLImageSink *sink = GST_GLIMAGE_SINK (navigation);
  GstEvent *event = NULL;
  GstPad *pad = NULL;
396
  GstGLWindow *window;
397 398 399
  guint width, height;
  gdouble x, y, xscale, yscale;

400 401 402 403
  if (!sink->context)
    return;

  window = gst_gl_context_get_window (sink->context);
404 405 406 407 408
  g_return_if_fail (GST_GL_IS_WINDOW (window));

  width = GST_VIDEO_SINK_WIDTH (sink);
  height = GST_VIDEO_SINK_HEIGHT (sink);
  gst_gl_window_get_surface_dimensions (window, &width, &height);
409 410 411 412

  event = gst_event_new_navigation (structure);

  pad = gst_pad_get_peer (GST_VIDEO_SINK_PAD (sink));
413 414 415 416 417 418 419 420 421 422 423 424 425 426
  /* Converting pointer coordinates to the non scaled geometry */
  if (width != GST_VIDEO_SINK_WIDTH (sink) &&
      width != 0 && gst_structure_get_double (structure, "pointer_x", &x)) {
    xscale = (gdouble) GST_VIDEO_SINK_WIDTH (sink) / width;
    gst_structure_set (structure, "pointer_x", G_TYPE_DOUBLE,
        (gdouble) x * xscale, NULL);
  }
  if (height != GST_VIDEO_SINK_HEIGHT (sink) &&
      height != 0 && gst_structure_get_double (structure, "pointer_y", &y)) {
    yscale = (gdouble) GST_VIDEO_SINK_HEIGHT (sink) / height;
    gst_structure_set (structure, "pointer_y", G_TYPE_DOUBLE,
        (gdouble) y * yscale, NULL);
  }

427 428 429 430 431

  if (GST_IS_PAD (pad) && GST_IS_EVENT (event))
    gst_pad_send_event (pad, event);

  gst_object_unref (pad);
432
  gst_object_unref (window);
433 434 435 436 437 438 439 440
}

static void
gst_glimage_sink_navigation_interface_init (GstNavigationInterface * iface)
{
  iface->send_event = gst_glimage_sink_navigation_send_event;
}

441 442 443 444
#define gst_glimage_sink_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstGLImageSink, gst_glimage_sink,
    GST_TYPE_VIDEO_SINK, G_IMPLEMENT_INTERFACE (GST_TYPE_VIDEO_OVERLAY,
        gst_glimage_sink_video_overlay_init);
445 446
    G_IMPLEMENT_INTERFACE (GST_TYPE_NAVIGATION,
        gst_glimage_sink_navigation_interface_init);
447 448 449 450 451 452 453 454 455
    GST_DEBUG_CATEGORY_INIT (gst_debug_glimage_sink, "glimagesink", 0,
        "OpenGL Video Sink"));

static void
gst_glimage_sink_class_init (GstGLImageSinkClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *gstelement_class;
  GstBaseSinkClass *gstbasesink_class;
456
  GstVideoSinkClass *gstvideosink_class;
457 458 459 460 461
  GstElementClass *element_class;

  gobject_class = (GObjectClass *) klass;
  gstelement_class = (GstElementClass *) klass;
  gstbasesink_class = (GstBaseSinkClass *) klass;
462
  gstvideosink_class = (GstVideoSinkClass *) klass;
463 464 465 466 467 468 469 470 471 472
  element_class = GST_ELEMENT_CLASS (klass);

  gobject_class->set_property = gst_glimage_sink_set_property;
  gobject_class->get_property = gst_glimage_sink_get_property;

  g_object_class_install_property (gobject_class, ARG_DISPLAY,
      g_param_spec_string ("display", "Display", "Display name",
          NULL, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  g_object_class_install_property (gobject_class, PROP_FORCE_ASPECT_RATIO,
473 474 475
      g_param_spec_boolean ("force-aspect-ratio", "Force aspect ratio",
          "When enabled, scaling will respect original aspect ratio",
          DEFAULT_FORCE_ASPECT_RATIO,
476 477 478 479
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  g_object_class_install_property (gobject_class, PROP_PIXEL_ASPECT_RATIO,
      gst_param_spec_fraction ("pixel-aspect-ratio", "Pixel Aspect Ratio",
480
          "The pixel aspect ratio of the device", 0, 1, G_MAXINT, 1, 1, 1,
481 482
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

Lubosz Sarnecki's avatar
Lubosz Sarnecki committed
483
  g_object_class_install_property (gobject_class, PROP_CONTEXT,
484
      g_param_spec_object ("context", "OpenGL context", "Get OpenGL context",
Lubosz Sarnecki's avatar
Lubosz Sarnecki committed
485 486
          GST_GL_TYPE_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));

487 488 489 490 491
  g_object_class_install_property (gobject_class, PROP_HANDLE_EVENTS,
      g_param_spec_boolean ("handle-events", "Handle XEvents",
          "When enabled, XEvents will be selected and handled", TRUE,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

492
  g_object_class_install_property (gobject_class, PROP_IGNORE_ALPHA,
493 494 495 496
      g_param_spec_boolean ("ignore-alpha", "Ignore Alpha",
          "When enabled, alpha will be ignored and converted to black", TRUE,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

497 498 499 500
  gst_element_class_set_metadata (element_class, "OpenGL video sink",
      "Sink/Video", "A videosink based on OpenGL",
      "Julien Isorce <julien.isorce@gmail.com>");

501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
  /**
   * GstGLImageSink::client-draw:
   * @object: the #GstGLImageSink
   * @texture: the #guint id of the texture.
   * @width: the #guint width of the texture.
   * @height: the #guint height of the texture.
   *
   * Will be emitted before actually drawing the texture.  The client should
   * redraw the surface/contents with the @texture, @width and @height and
   * and return %TRUE.
   *
   * Returns: whether the texture was redrawn by the signal.  If not, a
   *          default redraw will occur.
   */
  gst_glimage_sink_signals[CLIENT_DRAW_SIGNAL] =
      g_signal_new ("client-draw", G_TYPE_FROM_CLASS (klass),
      G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
518
      G_TYPE_BOOLEAN, 2, GST_GL_TYPE_CONTEXT, GST_TYPE_SAMPLE);
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534

  /**
   * GstGLImageSink::client-reshape:
   * @object: the #GstGLImageSink
   * @width: the #guint width of the texture.
   * @height: the #guint height of the texture.
   *
   * The client should resize the surface/window/viewport with the @width and
   * @height and return %TRUE.
   *
   * Returns: whether the content area was resized by the signal.  If not, a
   *          default viewport resize will occur.
   */
  gst_glimage_sink_signals[CLIENT_RESHAPE_SIGNAL] =
      g_signal_new ("client-reshape", G_TYPE_FROM_CLASS (klass),
      G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
Lubosz Sarnecki's avatar
Lubosz Sarnecki committed
535
      G_TYPE_BOOLEAN, 3, GST_GL_TYPE_CONTEXT, G_TYPE_UINT, G_TYPE_UINT);
536

537 538 539 540 541 542 543 544 545
  gst_element_class_add_pad_template (element_class,
      gst_static_pad_template_get (&gst_glimage_sink_template));

  gobject_class->finalize = gst_glimage_sink_finalize;

  gstelement_class->change_state = gst_glimage_sink_change_state;
  gstelement_class->set_context = gst_glimage_sink_set_context;
  gstbasesink_class->query = GST_DEBUG_FUNCPTR (gst_glimage_sink_query);
  gstbasesink_class->set_caps = gst_glimage_sink_set_caps;
546
  gstbasesink_class->get_caps = gst_glimage_sink_get_caps;
547
  gstbasesink_class->get_times = gst_glimage_sink_get_times;
548
  gstbasesink_class->prepare = gst_glimage_sink_prepare;
549 550
  gstbasesink_class->propose_allocation = gst_glimage_sink_propose_allocation;
  gstbasesink_class->stop = gst_glimage_sink_stop;
551 552 553

  gstvideosink_class->show_frame =
      GST_DEBUG_FUNCPTR (gst_glimage_sink_show_frame);
554 555 556 557 558 559 560 561 562
}

static void
gst_glimage_sink_init (GstGLImageSink * glimage_sink)
{
  glimage_sink->display_name = NULL;
  glimage_sink->window_id = 0;
  glimage_sink->new_window_id = 0;
  glimage_sink->display = NULL;
563
  glimage_sink->keep_aspect_ratio = TRUE;
564
  glimage_sink->par_n = 0;
565 566
  glimage_sink->par_d = 1;
  glimage_sink->pool = NULL;
567
  glimage_sink->stored_buffer = NULL;
568
  glimage_sink->redisplay_texture = 0;
569
  glimage_sink->handle_events = TRUE;
570
  glimage_sink->ignore_alpha = TRUE;
571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602

  g_mutex_init (&glimage_sink->drawing_lock);
}

static void
gst_glimage_sink_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstGLImageSink *glimage_sink;

  g_return_if_fail (GST_IS_GLIMAGE_SINK (object));

  glimage_sink = GST_GLIMAGE_SINK (object);

  switch (prop_id) {
    case ARG_DISPLAY:
    {
      g_free (glimage_sink->display_name);
      glimage_sink->display_name = g_strdup (g_value_get_string (value));
      break;
    }
    case PROP_FORCE_ASPECT_RATIO:
    {
      glimage_sink->keep_aspect_ratio = g_value_get_boolean (value);
      break;
    }
    case PROP_PIXEL_ASPECT_RATIO:
    {
      glimage_sink->par_n = gst_value_get_fraction_numerator (value);
      glimage_sink->par_d = gst_value_get_fraction_denominator (value);
      break;
    }
603 604 605 606
    case PROP_HANDLE_EVENTS:
      gst_glimage_sink_handle_events (GST_VIDEO_OVERLAY (glimage_sink),
          g_value_get_boolean (value));
      break;
607 608 609
    case PROP_IGNORE_ALPHA:
      glimage_sink->ignore_alpha = g_value_get_boolean (value);
      break;
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_glimage_sink_finalize (GObject * object)
{
  GstGLImageSink *glimage_sink;

  g_return_if_fail (GST_IS_GLIMAGE_SINK (object));

  glimage_sink = GST_GLIMAGE_SINK (object);

  g_mutex_clear (&glimage_sink->drawing_lock);

627 628 629 630 631
  if (glimage_sink->other_context) {
    gst_object_unref (glimage_sink->other_context);
    glimage_sink->other_context = NULL;
  }

632 633 634
  g_free (glimage_sink->display_name);

  GST_DEBUG ("finalized");
635
  G_OBJECT_CLASS (parent_class)->finalize (object);
636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
}

static void
gst_glimage_sink_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
  GstGLImageSink *glimage_sink;

  g_return_if_fail (GST_IS_GLIMAGE_SINK (object));

  glimage_sink = GST_GLIMAGE_SINK (object);

  switch (prop_id) {
    case ARG_DISPLAY:
      g_value_set_string (value, glimage_sink->display_name);
      break;
    case PROP_FORCE_ASPECT_RATIO:
      g_value_set_boolean (value, glimage_sink->keep_aspect_ratio);
      break;
    case PROP_PIXEL_ASPECT_RATIO:
      gst_value_set_fraction (value, glimage_sink->par_n, glimage_sink->par_d);
      break;
Lubosz Sarnecki's avatar
Lubosz Sarnecki committed
658 659
    case PROP_CONTEXT:
      g_value_set_object (value, glimage_sink->context);
660
      break;
661 662 663
    case PROP_HANDLE_EVENTS:
      g_value_set_boolean (value, glimage_sink->handle_events);
      break;
664 665 666
    case PROP_IGNORE_ALPHA:
      g_value_set_boolean (value, glimage_sink->ignore_alpha);
      break;
667 668 669 670 671 672
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
static void
gst_glimage_sink_key_event_cb (GstGLWindow * window, char *event_name, char
    *key_string, GstGLImageSink * gl_sink)
{
  GST_DEBUG_OBJECT (gl_sink, "glimagesink event %s key %s pressed", event_name,
      key_string);
  gst_navigation_send_key_event (GST_NAVIGATION (gl_sink),
      event_name, key_string);
}

static void
gst_glimage_sink_mouse_event_cb (GstGLWindow * window, char *event_name,
    int button, double posx, double posy, GstGLImageSink * gl_sink)
{
  GST_DEBUG_OBJECT (gl_sink, "glimagesink event %s at %g, %g", event_name, posx,
688
      posy);
689 690 691 692
  gst_navigation_send_mouse_event (GST_NAVIGATION (gl_sink),
      event_name, button, posx, posy);
}

693 694 695 696 697
static gboolean
_ensure_gl_setup (GstGLImageSink * gl_sink)
{
  GError *error = NULL;

698 699
  GST_DEBUG_OBJECT (gl_sink, "Ensuring setup");

700
  if (!gl_sink->context) {
701 702 703
    do {
      GstGLContext *other_context;
      GstGLWindow *window;
704

705 706
      if (gl_sink->context)
        gst_object_unref (gl_sink->context);
707

708 709 710
      GST_DEBUG_OBJECT (gl_sink,
          "No current context, creating one for %" GST_PTR_FORMAT,
          gl_sink->display);
711

712 713 714
      gl_sink->context = gst_gl_context_new (gl_sink->display);
      if (!gl_sink->context)
        goto context_creation_error;
715

716
      window = gst_gl_context_get_window (gl_sink->context);
717

718
      GST_DEBUG_OBJECT (gl_sink, "got window %" GST_PTR_FORMAT, window);
719

720 721
      if (!gl_sink->window_id && !gl_sink->new_window_id)
        gst_video_overlay_prepare_window_handle (GST_VIDEO_OVERLAY (gl_sink));
722

723 724 725
      GST_DEBUG_OBJECT (gl_sink,
          "window_id : %" G_GUINTPTR_FORMAT " , new_window_id : %"
          G_GUINTPTR_FORMAT, gl_sink->window_id, gl_sink->new_window_id);
726

727 728 729 730 731
      if (gl_sink->window_id != gl_sink->new_window_id) {
        gl_sink->window_id = gl_sink->new_window_id;
        GST_DEBUG_OBJECT (gl_sink, "Setting window handle on gl window");
        gst_gl_window_set_window_handle (window, gl_sink->window_id);
      }
732

733 734 735 736 737 738
      if (gl_sink->other_context) {
        other_context = gst_object_ref (gl_sink->other_context);
      } else {
        other_context =
            gst_gl_display_get_gl_context_for_thread (gl_sink->display, NULL);
      }
739

740 741 742 743 744
      GST_DEBUG_OBJECT (gl_sink,
          "creating context %" GST_PTR_FORMAT " from other context %"
          GST_PTR_FORMAT, gl_sink->context, other_context);

      if (!gst_gl_context_create (gl_sink->context, other_context, &error)) {
745 746
        if (other_context)
          gst_object_unref (other_context);
747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772
        gst_object_unref (window);
        goto context_error;
      }

      gst_gl_window_handle_events (window, gl_sink->handle_events);

      /* setup callbacks */
      gst_gl_window_set_resize_callback (window,
          GST_GL_WINDOW_RESIZE_CB (gst_glimage_sink_on_resize),
          gst_object_ref (gl_sink), (GDestroyNotify) gst_object_unref);
      gst_gl_window_set_draw_callback (window,
          GST_GL_WINDOW_CB (gst_glimage_sink_on_draw),
          gst_object_ref (gl_sink), (GDestroyNotify) gst_object_unref);
      gst_gl_window_set_close_callback (window,
          GST_GL_WINDOW_CB (gst_glimage_sink_on_close),
          gst_object_ref (gl_sink), (GDestroyNotify) gst_object_unref);
      gl_sink->key_sig_id = g_signal_connect (window, "key-event", G_CALLBACK
          (gst_glimage_sink_key_event_cb), gl_sink);
      gl_sink->mouse_sig_id =
          g_signal_connect (window, "mouse-event",
          G_CALLBACK (gst_glimage_sink_mouse_event_cb), gl_sink);

      if (other_context)
        gst_object_unref (other_context);
      gst_object_unref (window);
    } while (!gst_gl_display_add_context (gl_sink->display, gl_sink->context));
773 774
  } else
    GST_DEBUG_OBJECT (gl_sink, "Already have a context");
775 776 777

  return TRUE;

778 779 780 781 782 783 784
context_creation_error:
  {
    GST_ELEMENT_ERROR (gl_sink, RESOURCE, NOT_FOUND,
        ("Failed to create GL context"), (NULL));
    return FALSE;
  }

785 786 787 788
context_error:
  {
    GST_ELEMENT_ERROR (gl_sink, RESOURCE, NOT_FOUND, ("%s", error->message),
        (NULL));
789
    gst_object_unref (gl_sink->context);
790
    gl_sink->context = NULL;
791 792 793 794 795 796 797 798 799 800 801 802 803
    return FALSE;
  }
}

static gboolean
gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query)
{
  GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (bsink);
  gboolean res = FALSE;

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_CONTEXT:
    {
804 805 806 807 808
      const gchar *context_type;
      GstContext *context, *old_context;
      gboolean ret;

      ret =
809
          gst_gl_handle_context_query ((GstElement *) glimage_sink, query,
810
          &glimage_sink->display, &glimage_sink->other_context);
811 812
      if (glimage_sink->display)
        gst_gl_display_filter_gl_api (glimage_sink->display, SUPPORTED_GL_APIS);
813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836

      gst_query_parse_context_type (query, &context_type);

      if (g_strcmp0 (context_type, "gst.gl.local_context") == 0) {
        GstStructure *s;

        gst_query_parse_context (query, &old_context);

        if (old_context)
          context = gst_context_copy (old_context);
        else
          context = gst_context_new ("gst.gl.local_context", FALSE);

        s = gst_context_writable_structure (context);
        gst_structure_set (s, "context", GST_GL_TYPE_CONTEXT,
            glimage_sink->context, NULL);
        gst_query_set_context (query, context);
        gst_context_unref (context);

        ret = glimage_sink->context != NULL;
      }
      GST_DEBUG_OBJECT (glimage_sink, "context query of type %s %i",
          context_type, ret);

837
      return ret;
838
    }
839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856
    case GST_QUERY_DRAIN:
    {
      GstBuffer *buf = NULL;

      GST_GLIMAGE_SINK_LOCK (glimage_sink);
      glimage_sink->redisplay_texture = 0;
      buf = glimage_sink->stored_buffer;
      glimage_sink->stored_buffer = NULL;
      GST_GLIMAGE_SINK_UNLOCK (glimage_sink);

      if (buf)
        gst_buffer_unref (buf);

      gst_buffer_replace (&glimage_sink->next_buffer, NULL);

      res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query);
      break;
    }
857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882
    default:
      res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query);
      break;
  }

  return res;
}

static gboolean
gst_glimage_sink_stop (GstBaseSink * bsink)
{
  GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (bsink);

  if (glimage_sink->pool) {
    gst_object_unref (glimage_sink->pool);
    glimage_sink->pool = NULL;
  }

  return TRUE;
}

static void
gst_glimage_sink_set_context (GstElement * element, GstContext * context)
{
  GstGLImageSink *gl_sink = GST_GLIMAGE_SINK (element);

Lubosz Sarnecki's avatar
Lubosz Sarnecki committed
883 884
  gst_gl_handle_set_context (element, context, &gl_sink->display,
      &gl_sink->other_context);
885 886 887

  if (gl_sink->display)
    gst_gl_display_filter_gl_api (gl_sink->display, SUPPORTED_GL_APIS);
888 889 890 891 892 893 894 895
}

static GstStateChangeReturn
gst_glimage_sink_change_state (GstElement * element, GstStateChange transition)
{
  GstGLImageSink *glimage_sink;
  GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;

896 897 898
  GST_DEBUG ("changing state: %s => %s",
      gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)),
      gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition)));
899 900 901 902 903

  glimage_sink = GST_GLIMAGE_SINK (element);

  switch (transition) {
    case GST_STATE_CHANGE_NULL_TO_READY:
904 905 906 907 908
      if (!gst_gl_ensure_element_data (glimage_sink, &glimage_sink->display,
              &glimage_sink->other_context))
        return GST_STATE_CHANGE_FAILURE;

      gst_gl_display_filter_gl_api (glimage_sink->display, SUPPORTED_GL_APIS);
909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932
      break;
    case GST_STATE_CHANGE_READY_TO_PAUSED:
      g_atomic_int_set (&glimage_sink->to_quit, 0);
      break;
    case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
      break;
    default:
      break;
  }

  ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
  if (ret == GST_STATE_CHANGE_FAILURE)
    return ret;

  switch (transition) {
    case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
      break;
    case GST_STATE_CHANGE_PAUSED_TO_READY:
    {
      /* mark the redisplay_texture as unavailable (=0)
       * to avoid drawing
       */
      GST_GLIMAGE_SINK_LOCK (glimage_sink);
      glimage_sink->redisplay_texture = 0;
933 934 935 936
      if (glimage_sink->stored_buffer) {
        gst_buffer_unref (glimage_sink->stored_buffer);
        glimage_sink->stored_buffer = NULL;
      }
937
      GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
938
      gst_buffer_replace (&glimage_sink->next_buffer, NULL);
939 940

      glimage_sink->window_id = 0;
941
      /* but do not reset glimage_sink->new_window_id */
942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957

      GST_VIDEO_SINK_WIDTH (glimage_sink) = 1;
      GST_VIDEO_SINK_HEIGHT (glimage_sink) = 1;
      if (glimage_sink->context) {
        GstGLWindow *window = gst_gl_context_get_window (glimage_sink->context);

        gst_gl_window_send_message (window,
            GST_GL_WINDOW_CB (gst_glimage_sink_cleanup_glthread), glimage_sink);

        gst_gl_window_set_resize_callback (window, NULL, NULL, NULL);
        gst_gl_window_set_draw_callback (window, NULL, NULL, NULL);
        gst_gl_window_set_close_callback (window, NULL, NULL, NULL);

        gst_object_unref (window);
        gst_object_unref (glimage_sink->context);
        glimage_sink->context = NULL;
958 959 960
      }

      if (glimage_sink->display) {
961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997
        gst_object_unref (glimage_sink->display);
        glimage_sink->display = NULL;
      }
      break;
    }
    case GST_STATE_CHANGE_READY_TO_NULL:
      break;
    default:
      break;
  }

  return ret;
}

static void
gst_glimage_sink_get_times (GstBaseSink * bsink, GstBuffer * buf,
    GstClockTime * start, GstClockTime * end)
{
  GstGLImageSink *glimagesink;

  glimagesink = GST_GLIMAGE_SINK (bsink);

  if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) {
    *start = GST_BUFFER_TIMESTAMP (buf);
    if (GST_BUFFER_DURATION_IS_VALID (buf))
      *end = *start + GST_BUFFER_DURATION (buf);
    else {
      if (GST_VIDEO_INFO_FPS_N (&glimagesink->info) > 0) {
        *end = *start +
            gst_util_uint64_scale_int (GST_SECOND,
            GST_VIDEO_INFO_FPS_D (&glimagesink->info),
            GST_VIDEO_INFO_FPS_N (&glimagesink->info));
      }
    }
  }
}

998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
static GstCaps *
gst_glimage_sink_get_caps (GstBaseSink * bsink, GstCaps * filter)
{
  GstCaps *tmp = NULL;
  GstCaps *result = NULL;

  tmp = gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA");

  if (filter) {
    result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
    gst_caps_unref (tmp);
  } else {
    result = tmp;
  }

  GST_DEBUG_OBJECT (bsink, "returning caps: %" GST_PTR_FORMAT, result);

  return result;
}

1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
static gboolean
gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
{
  GstGLImageSink *glimage_sink;
  gint width;
  gint height;
  gboolean ok;
  gint par_n, par_d;
  gint display_par_n, display_par_d;
  guint display_ratio_num, display_ratio_den;
  GstVideoInfo vinfo;

1030
  GST_DEBUG_OBJECT (bsink, "set caps with %" GST_PTR_FORMAT, caps);
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087

  glimage_sink = GST_GLIMAGE_SINK (bsink);

  ok = gst_video_info_from_caps (&vinfo, caps);
  if (!ok)
    return FALSE;

  width = GST_VIDEO_INFO_WIDTH (&vinfo);
  height = GST_VIDEO_INFO_HEIGHT (&vinfo);

  par_n = GST_VIDEO_INFO_PAR_N (&vinfo);
  par_d = GST_VIDEO_INFO_PAR_D (&vinfo);

  if (!par_n)
    par_n = 1;

  /* get display's PAR */
  if (glimage_sink->par_n != 0 && glimage_sink->par_d != 0) {
    display_par_n = glimage_sink->par_n;
    display_par_d = glimage_sink->par_d;
  } else {
    display_par_n = 1;
    display_par_d = 1;
  }

  ok = gst_video_calculate_display_ratio (&display_ratio_num,
      &display_ratio_den, width, height, par_n, par_d, display_par_n,
      display_par_d);

  if (!ok)
    return FALSE;

  GST_TRACE ("PAR: %u/%u DAR:%u/%u", par_n, par_d, display_par_n,
      display_par_d);

  if (height % display_ratio_den == 0) {
    GST_DEBUG ("keeping video height");
    GST_VIDEO_SINK_WIDTH (glimage_sink) = (guint)
        gst_util_uint64_scale_int (height, display_ratio_num,
        display_ratio_den);
    GST_VIDEO_SINK_HEIGHT (glimage_sink) = height;
  } else if (width % display_ratio_num == 0) {
    GST_DEBUG ("keeping video width");
    GST_VIDEO_SINK_WIDTH (glimage_sink) = width;
    GST_VIDEO_SINK_HEIGHT (glimage_sink) = (guint)
        gst_util_uint64_scale_int (width, display_ratio_den, display_ratio_num);
  } else {
    GST_DEBUG ("approximating while keeping video height");
    GST_VIDEO_SINK_WIDTH (glimage_sink) = (guint)
        gst_util_uint64_scale_int (height, display_ratio_num,
        display_ratio_den);
    GST_VIDEO_SINK_HEIGHT (glimage_sink) = height;
  }
  GST_DEBUG ("scaling to %dx%d", GST_VIDEO_SINK_WIDTH (glimage_sink),
      GST_VIDEO_SINK_HEIGHT (glimage_sink));

  glimage_sink->info = vinfo;
1088 1089 1090
  if (!_ensure_gl_setup (glimage_sink))
    return FALSE;

1091
  glimage_sink->caps_change = TRUE;
1092

1093 1094 1095 1096
  return TRUE;
}

static GstFlowReturn
1097
gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
1098 1099
{
  GstGLImageSink *glimage_sink;
1100
  GstVideoFrame gl_frame;
1101 1102 1103

  glimage_sink = GST_GLIMAGE_SINK (bsink);

1104 1105
  GST_TRACE ("preparing buffer:%p", buf);

1106 1107 1108 1109 1110 1111 1112 1113
  if (GST_VIDEO_SINK_WIDTH (glimage_sink) < 1 ||
      GST_VIDEO_SINK_HEIGHT (glimage_sink) < 1) {
    return GST_FLOW_NOT_NEGOTIATED;
  }

  if (!_ensure_gl_setup (glimage_sink))
    return GST_FLOW_NOT_NEGOTIATED;

1114
  if (!gst_video_frame_map (&gl_frame, &glimage_sink->info, buf,
1115 1116 1117 1118 1119 1120
          GST_MAP_READ | GST_MAP_GL)) {
    goto upload_failed;
  }

  glimage_sink->next_tex = *(guint *) gl_frame.data[0];

1121
  gst_buffer_replace (&glimage_sink->next_buffer, buf);
1122

1123 1124
  gst_video_frame_unmap (&gl_frame);

1125 1126 1127 1128 1129 1130 1131 1132 1133
  if (glimage_sink->window_id != glimage_sink->new_window_id) {
    GstGLWindow *window = gst_gl_context_get_window (glimage_sink->context);

    glimage_sink->window_id = glimage_sink->new_window_id;
    gst_gl_window_set_window_handle (window, glimage_sink->window_id);

    gst_object_unref (window);
  }

1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
  return GST_FLOW_OK;

upload_failed:
  {
    GST_ELEMENT_ERROR (glimage_sink, RESOURCE, NOT_FOUND,
        ("%s", "Failed to upload buffer"), (NULL));
    return GST_FLOW_ERROR;
  }
}

static GstFlowReturn
1145
gst_glimage_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf)
1146 1147
{
  GstGLImageSink *glimage_sink;
1148
  GstBuffer *stored_buffer;
1149 1150 1151

  GST_TRACE ("rendering buffer:%p", buf);

1152
  glimage_sink = GST_GLIMAGE_SINK (vsink);
1153 1154 1155

  GST_TRACE ("redisplay texture:%u of size:%ux%u, window size:%ux%u",
      glimage_sink->next_tex, GST_VIDEO_INFO_WIDTH (&glimage_sink->info),
1156 1157 1158 1159 1160 1161
      GST_VIDEO_INFO_HEIGHT (&glimage_sink->info),
      GST_VIDEO_SINK_WIDTH (glimage_sink),
      GST_VIDEO_SINK_HEIGHT (glimage_sink));

  /* Avoid to release the texture while drawing */
  GST_GLIMAGE_SINK_LOCK (glimage_sink);
1162
  glimage_sink->redisplay_texture = glimage_sink->next_tex;
1163
  stored_buffer = glimage_sink->stored_buffer;
1164
  glimage_sink->stored_buffer = gst_buffer_ref (glimage_sink->next_buffer);
1165 1166 1167 1168 1169 1170 1171 1172
  GST_GLIMAGE_SINK_UNLOCK (glimage_sink);

  /* Ask the underlying window to redraw its content */
  if (!gst_glimage_sink_redisplay (glimage_sink))
    goto redisplay_failed;

  GST_TRACE ("post redisplay");

1173 1174 1175
  if (stored_buffer)
    gst_buffer_unref (stored_buffer);

1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
  if (g_atomic_int_get (&glimage_sink->to_quit) != 0) {
    GST_ELEMENT_ERROR (glimage_sink, RESOURCE, NOT_FOUND,
        ("%s", gst_gl_context_get_error ()), (NULL));
    return GST_FLOW_ERROR;
  }

  return GST_FLOW_OK;

/* ERRORS */
redisplay_failed:
  {
    GST_ELEMENT_ERROR (glimage_sink, RESOURCE, NOT_FOUND,
        ("%s", gst_gl_context_get_error ()), (NULL));
    return GST_FLOW_ERROR;
  }
}

static void
gst_glimage_sink_video_overlay_init (GstVideoOverlayInterface * iface)
{
  iface->set_window_handle = gst_glimage_sink_set_window_handle;
1197
  iface->handle_events = gst_glimage_sink_handle_events;
1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
  iface->expose = gst_glimage_sink_expose;
}

static void
gst_glimage_sink_set_window_handle (GstVideoOverlay * overlay, guintptr id)
{
  GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (overlay);
  guintptr window_id = (guintptr) id;

  g_return_if_fail (GST_IS_GLIMAGE_SINK (overlay));

  GST_DEBUG ("set_xwindow_id %" G_GUINT64_FORMAT, (guint64) window_id);

  glimage_sink->new_window_id = window_id;
}


static void
gst_glimage_sink_expose (GstVideoOverlay * overlay)
{
  GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (overlay);

  /* redisplay opengl scene */
  if (glimage_sink->display && glimage_sink->window_id) {

    if (glimage_sink->window_id != glimage_sink->new_window_id) {
      GstGLWindow *window = gst_gl_context_get_window (glimage_sink->context);

      glimage_sink->window_id = glimage_sink->new_window_id;
      gst_gl_window_set_window_handle (window, glimage_sink->window_id);

      gst_object_unref (window);
    }

    gst_glimage_sink_redisplay (glimage_sink);
  }
}

1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
static void
gst_glimage_sink_handle_events (GstVideoOverlay * overlay,
    gboolean handle_events)
{
  GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (overlay);

  glimage_sink->handle_events = handle_events;
  if (G_LIKELY (glimage_sink->context)) {
    GstGLWindow *window;
    window = gst_gl_context_get_window (glimage_sink->context);
    gst_gl_window_handle_events (window, handle_events);
    gst_object_unref (window);
  }
}

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
static gboolean
gst_glimage_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query)
{
  GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (bsink);
  GstStructure *config;
  GstCaps *caps;
  guint size;
  gboolean need_pool;

  if (!_ensure_gl_setup (glimage_sink))
    return FALSE;

  gst_query_parse_allocation (query, &caps, &need_pool);

  if (caps == NULL)
    goto no_caps;

1268
  if (need_pool) {
1269 1270 1271 1272 1273 1274 1275 1276
    GstVideoInfo info;

    if (!gst_video_info_from_caps (&info, caps))
      goto invalid_caps;

    /* the normal size of a frame */
    size = info.size;

1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
    if (glimage_sink->pool) {
      GstCaps *pcaps;

      /* we had a pool, check caps */
      GST_DEBUG_OBJECT (glimage_sink, "check existing pool caps");
      config = gst_buffer_pool_get_config (glimage_sink->pool);
      gst_buffer_pool_config_get_params (config, &pcaps, &size, NULL, NULL);

      if (!gst_caps_is_equal (caps, pcaps)) {
        GST_DEBUG_OBJECT (glimage_sink, "pool has different caps");
        /* different caps, we can't use this pool */
        gst_object_unref (glimage_sink->pool);
        glimage_sink->pool = NULL;
      }
      gst_structure_free (config);
    }

    if (glimage_sink->pool == NULL) {
      GST_DEBUG_OBJECT (glimage_sink, "create new pool");

      glimage_sink->pool = gst_gl_buffer_pool_new (glimage_sink->context);
      config = gst_buffer_pool_get_config (glimage_sink->pool);
      gst_buffer_pool_config_set_params (config, caps, size, 0, 0);

      if (!gst_buffer_pool_set_config (glimage_sink->pool, config))
        goto config_failed;
    }

    /* we need at least 2 buffer because we hold on to the last one */
    gst_query_add_allocation_pool (query, glimage_sink->pool, size, 2, 0);
1307
  }
1308

1309 1310
  if (glimage_sink->context->gl_vtable->FenceSync)
    gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0);
1311 1312 1313 1314 1315 1316

  return TRUE;

  /* ERRORS */
no_caps:
  {
1317
    GST_WARNING_OBJECT (bsink, "no caps specified");
1318 1319 1320 1321
    return FALSE;
  }
invalid_caps:
  {
1322
    GST_WARNING_OBJECT (bsink, "invalid caps specified");
1323 1324 1325 1326
    return FALSE;
  }
config_failed:
  {
1327
    GST_WARNING_OBJECT (bsink, "failed setting config");
1328 1329 1330 1331
    return FALSE;
  }
}

1332 1333 1334 1335 1336 1337 1338 1339 1340
/* *INDENT-OFF* */
static const GLfloat vertices[] = {
     1.0f,  1.0f, 0.0f, 1.0f, 0.0f,
    -1.0f,  1.0f, 0.0f, 0.0f, 0.0f,
    -1.0f, -1.0f, 0.0f, 0.0f, 1.0f,
     1.0f, -1.0f, 0.0f, 1.0f, 1.0f
};
/* *INDENT-ON* */

1341
static void
1342
_bind_buffer (GstGLImageSink * gl_sink)
1343
{
1344 1345 1346 1347 1348 1349 1350
  const GstGLFuncs *gl = gl_sink->context->gl_vtable;

  gl->BindBuffer (GL_ARRAY_BUFFER, gl_sink->vertex_buffer);
  gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices,
      GL_STATIC_DRAW);

  /* Load the vertex position */
1351
  gl->VertexAttribPointer (gl_sink->attr_position, 3, GL_FLOAT, GL_FALSE,
1352 1353 1354
      5 * sizeof (GLfloat), (void *) 0);

  /* Load the texture coordinate */
1355
  gl->VertexAttribPointer (gl_sink->attr_texture, 2, GL_FLOAT, GL_FALSE,
1356 1357
      5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));

1358 1359 1360 1361 1362 1363 1364 1365
  gl->EnableVertexAttribArray (gl_sink->attr_position);
  gl->EnableVertexAttribArray (gl_sink->attr_texture);
}

static void
_unbind_buffer (GstGLImageSink * gl_sink)
{
  const GstGLFuncs *gl = gl_sink->context->gl_vtable;
1366 1367

  gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393

  gl->DisableVertexAttribArray (gl_sink->attr_position);
  gl->DisableVertexAttribArray (gl_sink->attr_texture);
}

/* Called in the gl thread */
static void
gst_glimage_sink_thread_init_redisplay (GstGLImageSink * gl_sink)
{
  const GstGLFuncs *gl = gl_sink->context->gl_vtable;

  gl_sink->redisplay_shader = gst_gl_shader_new (gl_sink->context);

  if (!gst_gl_shader_compile_with_default_vf_and_check
      (gl_sink->redisplay_shader, &gl_sink->attr_position,
          &gl_sink->attr_texture))
    gst_glimage_sink_cleanup_glthread (gl_sink);

  if (gl->GenVertexArrays) {
    gl->GenVertexArrays (1, &gl_sink->vao);
    gl->BindVertexArray (gl_sink->vao);
  }

  gl->GenBuffers (1, &gl_sink->vertex_buffer);
  _bind_buffer (gl_sink);

1394
  if (gl->GenVertexArrays) {
1395
    gl->BindVertexArray (0);
1396 1397 1398 1399
    gl->BindBuffer (GL_ARRAY_BUFFER, 0);
  } else {
    _unbind_buffer (gl_sink);
  }
1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
}

static void
gst_glimage_sink_cleanup_glthread (GstGLImageSink * gl_sink)
{
  const GstGLFuncs *gl = gl_sink->context->gl_vtable;

  if (gl_sink->redisplay_shader) {
    gst_object_unref (gl_sink->redisplay_shader);
    gl_sink->redisplay_shader = NULL;
  }

1412
  if (gl_sink->vao) {
1413
    gl->DeleteVertexArrays (1, &gl_sink->vao);
1414 1415
    gl_sink->vao = 0;
  }
1416 1417 1418
}

static void
1419
gst_glimage_sink_on_resize (GstGLImageSink * gl_sink, gint width, gint height)
1420 1421 1422 1423 1424
{
  /* Here gl_sink members (ex:gl_sink->info) have a life time of set_caps.
   * It means that they cannot not change between two set_caps
   */
  const GstGLFuncs *gl = gl_sink->context->gl_vtable;
1425
  gboolean do_reshape;
1426 1427 1428 1429

  GST_TRACE ("GL Window resized to %ux%u", width, height);

  /* check if a client reshape callback is registered */
1430
  g_signal_emit (gl_sink, gst_glimage_sink_signals[CLIENT_RESHAPE_SIGNAL], 0,
Lubosz Sarnecki's avatar
Lubosz Sarnecki committed
1431
      gl_sink->context, width, height, &do_reshape);
1432

1433 1434 1435
  width = MAX (1, width);
  height = MAX (1, height);

1436 1437 1438
  gl_sink->window_width = width;
  gl_sink->window_height = height;

1439
  /* default reshape */
1440
  if (!do_reshape) {
1441 1442 1443 1444 1445
    if (gl_sink->keep_aspect_ratio) {
      GstVideoRectangle src, dst, result;

      src.x = 0;
      src.y = 0;
1446 1447
      src.w = GST_VIDEO_SINK_WIDTH (gl_sink);
      src.h = GST_VIDEO_SINK_HEIGHT (gl_sink);
1448 1449 1450 14