gstwaylandsink.c 31.2 KB
Newer Older
1
/* GStreamer Wayland video sink
2
 *
3 4
 * Copyright (C) 2011 Intel Corporation
 * Copyright (C) 2011 Sreerenj Balachandran <sreerenj.balachandran@intel.com>
Wim Taymans's avatar
Wim Taymans committed
5
 * Copyright (C) 2012 Wim Taymans <wim.taymans@gmail.com>
6
 * Copyright (C) 2014 Collabora Ltd.
Wim Taymans's avatar
Wim Taymans committed
7
 *
8 9 10 11 12 13 14 15 16 17 18 19
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the Free
20 21
 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301 USA.
22 23
 */

24 25
/**
 * SECTION:element-waylandsink
26
 * @title: waylandsink
27 28 29
 *
 *  The waylandsink is creating its own window and render the decoded video frames to that.
 *  Setup the Wayland environment as described in
30 31 32
 *  [Wayland](http://wayland.freedesktop.org/building.html) home page.
 *
 *  The current implementation is based on weston compositor.
33
 *
34
 * ## Example pipelines
35
 * |[
36
 * gst-launch-1.0 -v videotestsrc ! waylandsink
37
 * ]| test the video rendering in wayland
38
 *
39
 */
40 41 42 43 44 45

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include "gstwaylandsink.h"
46
#include "wlvideoformat.h"
47
#include "wlbuffer.h"
48
#include "wlshmallocator.h"
49
#include "wllinuxdmabuf.h"
50

51 52 53
#include <gst/wayland/wayland.h>
#include <gst/video/videooverlay.h>

54 55 56 57 58 59 60 61 62 63 64
/* signals */
enum
{
  SIGNAL_0,
  LAST_SIGNAL
};

/* Properties */
enum
{
  PROP_0,
65 66
  PROP_DISPLAY,
  PROP_FULLSCREEN
67 68 69 70 71
};

GST_DEBUG_CATEGORY (gstwayland_debug);
#define GST_CAT_DEFAULT gstwayland_debug

72 73 74 75 76
#define WL_VIDEO_FORMATS \
    "{ BGRx, BGRA, RGBx, xBGR, xRGB, RGBA, ABGR, ARGB, RGB, BGR, " \
    "RGB16, BGR16, YUY2, YVYU, UYVY, AYUV, NV12, NV21, NV16, " \
    "YUV9, YVU9, Y41B, I420, YV12, Y42B, v308 }"

77 78 79
static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
    GST_PAD_SINK,
    GST_PAD_ALWAYS,
80 81 82
    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (WL_VIDEO_FORMATS) ";"
        GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_DMABUF,
            WL_VIDEO_FORMATS))
83
    );
84

85 86 87 88 89
static void gst_wayland_sink_get_property (GObject * object,
    guint prop_id, GValue * value, GParamSpec * pspec);
static void gst_wayland_sink_set_property (GObject * object,
    guint prop_id, const GValue * value, GParamSpec * pspec);
static void gst_wayland_sink_finalize (GObject * object);
90 91 92

static GstStateChangeReturn gst_wayland_sink_change_state (GstElement * element,
    GstStateChange transition);
93 94
static void gst_wayland_sink_set_context (GstElement * element,
    GstContext * context);
95

Wim Taymans's avatar
Wim Taymans committed
96 97
static GstCaps *gst_wayland_sink_get_caps (GstBaseSink * bsink,
    GstCaps * filter);
98
static gboolean gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps);
99 100
static gboolean
gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query);
101
static gboolean gst_wayland_sink_show_frame (GstVideoSink * bsink,
102 103
    GstBuffer * buffer);

104 105 106 107 108
/* VideoOverlay interface */
static void gst_wayland_sink_videooverlay_init (GstVideoOverlayInterface *
    iface);
static void gst_wayland_sink_set_window_handle (GstVideoOverlay * overlay,
    guintptr handle);
109 110
static void gst_wayland_sink_set_render_rectangle (GstVideoOverlay * overlay,
    gint x, gint y, gint w, gint h);
111 112 113 114 115
static void gst_wayland_sink_expose (GstVideoOverlay * overlay);

/* WaylandVideo interface */
static void gst_wayland_sink_waylandvideo_init (GstWaylandVideoInterface *
    iface);
116 117
static void gst_wayland_sink_begin_geometry_change (GstWaylandVideo * video);
static void gst_wayland_sink_end_geometry_change (GstWaylandVideo * video);
118 119 120 121 122 123 124

#define gst_wayland_sink_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstWaylandSink, gst_wayland_sink, GST_TYPE_VIDEO_SINK,
    G_IMPLEMENT_INTERFACE (GST_TYPE_VIDEO_OVERLAY,
        gst_wayland_sink_videooverlay_init)
    G_IMPLEMENT_INTERFACE (GST_TYPE_WAYLAND_VIDEO,
        gst_wayland_sink_waylandvideo_init));
125

126 127 128 129 130
/* A tiny GstVideoBufferPool subclass that modify the options to remove
 * VideoAlignment. To support VideoAlignment we would need to pass the padded
 * width/height + stride and use the viewporter interface to crop, a bit like
 * we use to do with XV. It would still be quite limited. It's a bit retro,
 * hopefully there will be a better Wayland interface in the future. */
131 132 133

GType gst_wayland_pool_get_type (void);

134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
typedef struct
{
  GstVideoBufferPool parent;
} GstWaylandPool;

typedef struct
{
  GstVideoBufferPoolClass parent;
} GstWaylandPoolClass;

G_DEFINE_TYPE (GstWaylandPool, gst_wayland_pool, GST_TYPE_VIDEO_BUFFER_POOL);

static const gchar **
gst_wayland_pool_get_options (GstBufferPool * pool)
{
  static const gchar *options[] = { GST_BUFFER_POOL_OPTION_VIDEO_META, NULL };
  return options;
}

static void
gst_wayland_pool_class_init (GstWaylandPoolClass * klass)
{
  GstBufferPoolClass *pool_class = GST_BUFFER_POOL_CLASS (klass);
  pool_class->get_options = gst_wayland_pool_get_options;
}

static void
gst_wayland_pool_init (GstWaylandPool * pool)
{
}

165
static void
166
gst_wayland_sink_class_init (GstWaylandSinkClass * klass)
167 168
{
  GObjectClass *gobject_class;
Wim Taymans's avatar
Wim Taymans committed
169
  GstElementClass *gstelement_class;
170
  GstBaseSinkClass *gstbasesink_class;
171
  GstVideoSinkClass *gstvideosink_class;
172 173

  gobject_class = (GObjectClass *) klass;
Wim Taymans's avatar
Wim Taymans committed
174
  gstelement_class = (GstElementClass *) klass;
175
  gstbasesink_class = (GstBaseSinkClass *) klass;
176
  gstvideosink_class = (GstVideoSinkClass *) klass;
177 178 179 180 181

  gobject_class->set_property = gst_wayland_sink_set_property;
  gobject_class->get_property = gst_wayland_sink_get_property;
  gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_wayland_sink_finalize);

182
  gst_element_class_add_static_pad_template (gstelement_class, &sink_template);
Wim Taymans's avatar
Wim Taymans committed
183

184
  gst_element_class_set_static_metadata (gstelement_class,
Wim Taymans's avatar
Wim Taymans committed
185 186
      "wayland video sink", "Sink/Video",
      "Output to wayland surface",
187 188
      "Sreerenj Balachandran <sreerenj.balachandran@intel.com>, "
      "George Kiagiadakis <george.kiagiadakis@collabora.com>");
Wim Taymans's avatar
Wim Taymans committed
189

190 191
  gstelement_class->change_state =
      GST_DEBUG_FUNCPTR (gst_wayland_sink_change_state);
192 193
  gstelement_class->set_context =
      GST_DEBUG_FUNCPTR (gst_wayland_sink_set_context);
194

195 196
  gstbasesink_class->get_caps = GST_DEBUG_FUNCPTR (gst_wayland_sink_get_caps);
  gstbasesink_class->set_caps = GST_DEBUG_FUNCPTR (gst_wayland_sink_set_caps);
197 198
  gstbasesink_class->propose_allocation =
      GST_DEBUG_FUNCPTR (gst_wayland_sink_propose_allocation);
199 200 201

  gstvideosink_class->show_frame =
      GST_DEBUG_FUNCPTR (gst_wayland_sink_show_frame);
202

203 204
  g_object_class_install_property (gobject_class, PROP_DISPLAY,
      g_param_spec_string ("display", "Wayland Display name", "Wayland "
205
          "display name to connect to, if not supplied via the GstContext",
206
          NULL, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
207 208 209 210 211

  g_object_class_install_property (gobject_class, PROP_FULLSCREEN,
      g_param_spec_boolean ("fullscreen", "Fullscreen",
          "Whether the surface should be made fullscreen ", FALSE,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
212 213 214
}

static void
Wim Taymans's avatar
Wim Taymans committed
215
gst_wayland_sink_init (GstWaylandSink * sink)
216
{
217
  g_mutex_init (&sink->display_lock);
218
  g_mutex_init (&sink->render_lock);
219 220
}

221 222 223 224 225 226 227 228 229 230 231 232
static void
gst_wayland_sink_set_fullscreen (GstWaylandSink * sink, gboolean fullscreen)
{
  if (fullscreen == sink->fullscreen)
    return;

  g_mutex_lock (&sink->render_lock);
  sink->fullscreen = fullscreen;
  gst_wl_window_ensure_fullscreen (sink->window, fullscreen);
  g_mutex_unlock (&sink->render_lock);
}

233 234 235 236
static void
gst_wayland_sink_get_property (GObject * object,
    guint prop_id, GValue * value, GParamSpec * pspec)
{
237
  GstWaylandSink *sink = GST_WAYLAND_SINK (object);
238 239

  switch (prop_id) {
240
    case PROP_DISPLAY:
241
      GST_OBJECT_LOCK (sink);
242
      g_value_set_string (value, sink->display_name);
243
      GST_OBJECT_UNLOCK (sink);
244
      break;
245 246 247 248 249
    case PROP_FULLSCREEN:
      GST_OBJECT_LOCK (sink);
      g_value_set_boolean (value, sink->fullscreen);
      GST_OBJECT_UNLOCK (sink);
      break;
250 251 252 253 254 255 256 257 258 259
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_wayland_sink_set_property (GObject * object,
    guint prop_id, const GValue * value, GParamSpec * pspec)
{
260
  GstWaylandSink *sink = GST_WAYLAND_SINK (object);
261 262

  switch (prop_id) {
263
    case PROP_DISPLAY:
264
      GST_OBJECT_LOCK (sink);
265
      sink->display_name = g_value_dup_string (value);
266
      GST_OBJECT_UNLOCK (sink);
267
      break;
268 269 270 271 272
    case PROP_FULLSCREEN:
      GST_OBJECT_LOCK (sink);
      gst_wayland_sink_set_fullscreen (sink, g_value_get_boolean (value));
      GST_OBJECT_UNLOCK (sink);
      break;
273 274 275 276 277 278 279 280 281
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_wayland_sink_finalize (GObject * object)
{
282
  GstWaylandSink *sink = GST_WAYLAND_SINK (object);
283

284 285
  GST_DEBUG_OBJECT (sink, "Finalizing the sink..");

286 287
  if (sink->last_buffer)
    gst_buffer_unref (sink->last_buffer);
288
  if (sink->display)
289
    g_object_unref (sink->display);
290
  if (sink->window)
291
    g_object_unref (sink->window);
292 293 294
  if (sink->pool)
    gst_object_unref (sink->pool);

295
  g_free (sink->display_name);
296

297
  g_mutex_clear (&sink->display_lock);
298
  g_mutex_clear (&sink->render_lock);
299

300 301 302
  G_OBJECT_CLASS (parent_class)->finalize (object);
}

303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321
/* must be called with the display_lock */
static void
gst_wayland_sink_set_display_from_context (GstWaylandSink * sink,
    GstContext * context)
{
  struct wl_display *display;
  GError *error = NULL;

  display = gst_wayland_display_handle_context_get_handle (context);
  sink->display = gst_wl_display_new_existing (display, FALSE, &error);

  if (error) {
    GST_ELEMENT_WARNING (sink, RESOURCE, OPEN_READ_WRITE,
        ("Could not set display handle"),
        ("Failed to use the external wayland display: '%s'", error->message));
    g_error_free (error);
  }
}

322 323
static gboolean
gst_wayland_sink_find_display (GstWaylandSink * sink)
324
{
325 326 327 328 329 330
  GstQuery *query;
  GstMessage *msg;
  GstContext *context = NULL;
  GError *error = NULL;
  gboolean ret = TRUE;

331 332
  g_mutex_lock (&sink->display_lock);

333 334 335 336 337
  if (!sink->display) {
    /* first query upstream for the needed display handle */
    query = gst_query_new_context (GST_WAYLAND_DISPLAY_HANDLE_CONTEXT_TYPE);
    if (gst_pad_peer_query (GST_VIDEO_SINK_PAD (sink), query)) {
      gst_query_parse_context (query, &context);
338
      gst_wayland_sink_set_display_from_context (sink, context);
339 340
    }
    gst_query_unref (query);
341

342 343 344 345
    if (G_LIKELY (!sink->display)) {
      /* now ask the application to set the display handle */
      msg = gst_message_new_need_context (GST_OBJECT_CAST (sink),
          GST_WAYLAND_DISPLAY_HANDLE_CONTEXT_TYPE);
346 347

      g_mutex_unlock (&sink->display_lock);
348 349 350
      gst_element_post_message (GST_ELEMENT_CAST (sink), msg);
      /* at this point we expect gst_wayland_sink_set_context
       * to get called and fill sink->display */
351
      g_mutex_lock (&sink->display_lock);
352 353

      if (!sink->display) {
354
        /* if the application didn't set a display, let's create it ourselves */
355
        GST_OBJECT_LOCK (sink);
356
        sink->display = gst_wl_display_new (sink->display_name, &error);
357 358
        GST_OBJECT_UNLOCK (sink);

359
        if (error) {
360 361 362 363
          GST_ELEMENT_WARNING (sink, RESOURCE, OPEN_READ_WRITE,
              ("Could not initialise Wayland output"),
              ("Failed to create GstWlDisplay: '%s'", error->message));
          g_error_free (error);
364
          ret = FALSE;
365 366
        }
      }
367 368 369
    }
  }

370 371
  g_mutex_unlock (&sink->display_lock);

372 373 374 375 376 377 378 379 380 381 382 383 384
  return ret;
}

static GstStateChangeReturn
gst_wayland_sink_change_state (GstElement * element, GstStateChange transition)
{
  GstWaylandSink *sink = GST_WAYLAND_SINK (element);
  GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;

  switch (transition) {
    case GST_STATE_CHANGE_NULL_TO_READY:
      if (!gst_wayland_sink_find_display (sink))
        return GST_STATE_CHANGE_FAILURE;
385 386 387 388 389 390 391 392 393 394 395
      break;
    default:
      break;
  }

  ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
  if (ret == GST_STATE_CHANGE_FAILURE)
    return ret;

  switch (transition) {
    case GST_STATE_CHANGE_PAUSED_TO_READY:
396 397 398 399 400 401
      gst_buffer_replace (&sink->last_buffer, NULL);
      if (sink->window) {
        if (gst_wl_window_is_toplevel (sink->window)) {
          g_clear_object (&sink->window);
        } else {
          /* remove buffer from surface, show nothing */
402
          gst_wl_window_render (sink->window, NULL, NULL);
403
        }
404 405 406
      }
      break;
    case GST_STATE_CHANGE_READY_TO_NULL:
407
      g_mutex_lock (&sink->display_lock);
408 409 410 411 412 413 414 415 416 417
      /* If we had a toplevel window, we most likely have our own connection
       * to the display too, and it is a good idea to disconnect and allow
       * potentially the application to embed us with GstVideoOverlay
       * (which requires to re-use the same display connection as the parent
       * surface). If we didn't have a toplevel window, then the display
       * connection that we have is definitely shared with the application
       * and it's better to keep it around (together with the window handle)
       * to avoid requesting them again from the application if/when we are
       * restarted (GstVideoOverlay behaves like that in other sinks)
       */
418
      if (sink->display && !sink->window) {     /* -> the window was toplevel */
419
        g_clear_object (&sink->display);
420 421 422
        g_mutex_lock (&sink->render_lock);
        sink->redraw_pending = FALSE;
        g_mutex_unlock (&sink->render_lock);
423
      }
424
      g_mutex_unlock (&sink->display_lock);
425
      g_clear_object (&sink->pool);
426 427 428 429 430 431 432 433
      break;
    default:
      break;
  }

  return ret;
}

434 435 436 437 438 439 440
static void
gst_wayland_sink_set_context (GstElement * element, GstContext * context)
{
  GstWaylandSink *sink = GST_WAYLAND_SINK (element);

  if (gst_context_has_context_type (context,
          GST_WAYLAND_DISPLAY_HANDLE_CONTEXT_TYPE)) {
441
    g_mutex_lock (&sink->display_lock);
442
    if (G_LIKELY (!sink->display)) {
443
      gst_wayland_sink_set_display_from_context (sink, context);
444
    } else {
445
      GST_WARNING_OBJECT (element, "changing display handle is not supported");
446 447 448
      g_mutex_unlock (&sink->display_lock);
      return;
    }
449
    g_mutex_unlock (&sink->display_lock);
450 451 452 453 454 455
  }

  if (GST_ELEMENT_CLASS (parent_class)->set_context)
    GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}

456
static GstCaps *
Wim Taymans's avatar
Wim Taymans committed
457
gst_wayland_sink_get_caps (GstBaseSink * bsink, GstCaps * filter)
458
{
459 460 461 462 463 464
  GstWaylandSink *sink;
  GstCaps *caps;

  sink = GST_WAYLAND_SINK (bsink);

  caps = gst_pad_get_pad_template_caps (GST_VIDEO_SINK_PAD (sink));
465
  caps = gst_caps_make_writable (caps);
466

467 468
  g_mutex_lock (&sink->display_lock);

469
  if (sink->display) {
470
    GValue shm_list = G_VALUE_INIT, dmabuf_list = G_VALUE_INIT;
471 472 473
    GValue value = G_VALUE_INIT;
    GArray *formats;
    gint i;
474
    guint fmt;
475

476 477
    g_value_init (&shm_list, GST_TYPE_LIST);
    g_value_init (&dmabuf_list, GST_TYPE_LIST);
478

479
    /* Add corresponding shm formats */
480
    formats = sink->display->shm_formats;
481
    for (i = 0; i < formats->len; i++) {
482
      g_value_init (&value, G_TYPE_STRING);
483
      fmt = g_array_index (formats, uint32_t, i);
484
      g_value_set_static_string (&value, gst_wl_shm_format_to_string (fmt));
485
      gst_value_list_append_and_take_value (&shm_list, &value);
486 487
    }

488
    gst_structure_take_value (gst_caps_get_structure (caps, 0), "format",
489 490 491 492 493 494 495 496 497 498 499 500 501
        &shm_list);

    /* Add corresponding dmabuf formats */
    formats = sink->display->dmabuf_formats;
    for (i = 0; i < formats->len; i++) {
      g_value_init (&value, G_TYPE_STRING);
      fmt = g_array_index (formats, uint32_t, i);
      g_value_set_static_string (&value, gst_wl_dmabuf_format_to_string (fmt));
      gst_value_list_append_and_take_value (&dmabuf_list, &value);
    }

    gst_structure_take_value (gst_caps_get_structure (caps, 1), "format",
        &dmabuf_list);
502 503 504 505

    GST_DEBUG_OBJECT (sink, "display caps: %" GST_PTR_FORMAT, caps);
  }

506 507
  g_mutex_unlock (&sink->display_lock);

508 509 510 511 512 513 514 515
  if (filter) {
    GstCaps *intersection;

    intersection =
        gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
    gst_caps_unref (caps);
    caps = intersection;
  }
516

517
  return caps;
518 519
}

520 521 522 523 524 525
static GstBufferPool *
gst_wayland_create_pool (GstWaylandSink * sink, GstCaps * caps)
{
  GstBufferPool *pool = NULL;
  GstStructure *structure;
  gsize size = sink->video_info.size;
526
  GstAllocator *alloc;
527

528
  pool = g_object_new (gst_wayland_pool_get_type (), NULL);
529 530

  structure = gst_buffer_pool_get_config (pool);
531
  gst_buffer_pool_config_set_params (structure, caps, size, 2, 0);
532

533 534
  alloc = gst_wl_shm_allocator_get ();
  gst_buffer_pool_config_set_allocator (structure, alloc, NULL);
535 536 537 538
  if (!gst_buffer_pool_set_config (pool, structure)) {
    g_object_unref (pool);
    pool = NULL;
  }
539
  g_object_unref (alloc);
540 541 542 543

  return pool;
}

544 545 546
static gboolean
gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
{
547
  GstWaylandSink *sink;
548 549
  gboolean use_dmabuf;
  GstVideoFormat format;
550 551

  sink = GST_WAYLAND_SINK (bsink);
552

553
  GST_DEBUG_OBJECT (sink, "set caps %" GST_PTR_FORMAT, caps);
554

555
  /* extract info from caps */
556
  if (!gst_video_info_from_caps (&sink->video_info, caps))
557
    goto invalid_format;
558

559 560
  format = GST_VIDEO_INFO_FORMAT (&sink->video_info);
  sink->video_info_changed = TRUE;
561

562 563 564 565 566
  /* create a new pool for the new caps */
  if (sink->pool)
    gst_object_unref (sink->pool);
  sink->pool = gst_wayland_create_pool (sink, caps);

567 568
  use_dmabuf = gst_caps_features_contains (gst_caps_get_features (caps, 0),
      GST_CAPS_FEATURE_MEMORY_DMABUF);
569

570 571 572
  /* validate the format base on the memory type. */
  if (use_dmabuf) {
    if (!gst_wl_display_check_format_for_dmabuf (sink->display, format))
573
      goto unsupported_format;
574 575
  } else if (!gst_wl_display_check_format_for_shm (sink->display, format)) {
    goto unsupported_format;
576 577 578
  }

  sink->use_dmabuf = use_dmabuf;
579

580
  return TRUE;
581

582 583
invalid_format:
  {
584
    GST_ERROR_OBJECT (sink,
585
        "Could not locate image format from caps %" GST_PTR_FORMAT, caps);
586
    return FALSE;
587
  }
588 589
unsupported_format:
  {
590
    GST_ERROR_OBJECT (sink, "Format %s is not available on the display",
591
        gst_video_format_to_string (format));
592
    return FALSE;
593
  }
594 595 596
}

static gboolean
597 598 599
gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query)
{
  GstWaylandSink *sink = GST_WAYLAND_SINK (bsink);
600 601 602
  GstCaps *caps;
  GstBufferPool *pool = NULL;
  gboolean need_pool;
603
  GstAllocator *alloc;
604

605
  gst_query_parse_allocation (query, &caps, &need_pool);
606

607 608 609
  if (need_pool)
    pool = gst_wayland_create_pool (sink, caps);

610 611
  gst_query_add_allocation_pool (query, pool, sink->video_info.size, 2, 0);
  if (pool)
612 613
    g_object_unref (pool);

614
  alloc = gst_wl_shm_allocator_get ();
615
  gst_query_add_allocation_param (query, alloc, NULL);
616
  gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL);
617
  g_object_unref (alloc);
618

619 620 621 622 623 624
  return TRUE;
}

static void
frame_redraw_callback (void *data, struct wl_callback *callback, uint32_t time)
{
625 626
  GstWaylandSink *sink = data;

627
  GST_LOG ("frame_redraw_cb");
628

629 630 631 632
  g_mutex_lock (&sink->render_lock);
  sink->redraw_pending = FALSE;
  g_mutex_unlock (&sink->render_lock);

633 634 635 636 637 638 639
  wl_callback_destroy (callback);
}

static const struct wl_callback_listener frame_callback_listener = {
  frame_redraw_callback
};

640
/* must be called with the render lock */
641
static void
642
render_last_buffer (GstWaylandSink * sink, gboolean redraw)
643
{
644
  GstWlBuffer *wlbuffer;
645
  const GstVideoInfo *info = NULL;
646 647 648
  struct wl_surface *surface;
  struct wl_callback *callback;

649
  wlbuffer = gst_buffer_get_wl_buffer (sink->last_buffer);
650 651
  surface = gst_wl_window_get_wl_surface (sink->window);

652
  sink->redraw_pending = TRUE;
653 654 655
  callback = wl_surface_frame (surface);
  wl_callback_add_listener (callback, &frame_callback_listener, sink);

656
  if (G_UNLIKELY (sink->video_info_changed && !redraw)) {
657 658 659 660
    info = &sink->video_info;
    sink->video_info_changed = FALSE;
  }
  gst_wl_window_render (sink->window, wlbuffer, info);
661 662
}

663 664 665 666 667 668 669 670 671 672
static void
on_window_closed (GstWlWindow * window, gpointer user_data)
{
  GstWaylandSink *sink = GST_WAYLAND_SINK (user_data);

  /* Handle window closure by posting an error on the bus */
  GST_ELEMENT_ERROR (sink, RESOURCE, NOT_FOUND,
      ("Output window was closed"), (NULL));
}

673
static GstFlowReturn
674
gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer)
675
{
676
  GstWaylandSink *sink = GST_WAYLAND_SINK (vsink);
Wim Taymans's avatar
Wim Taymans committed
677
  GstBuffer *to_render;
678
  GstWlBuffer *wlbuffer;
679 680
  GstVideoMeta *vmeta;
  GstVideoFormat format;
681
  GstVideoInfo old_vinfo;
682 683 684
  GstMemory *mem;
  struct wl_buffer *wbuf = NULL;

685
  GstFlowReturn ret = GST_FLOW_OK;
686

687
  g_mutex_lock (&sink->render_lock);
688

689
  GST_LOG_OBJECT (sink, "render buffer %p", buffer);
690

691 692 693 694 695 696 697
  if (G_UNLIKELY (!sink->window)) {
    /* ask for window handle. Unlock render_lock while doing that because
     * set_window_handle & friends will lock it in this context */
    g_mutex_unlock (&sink->render_lock);
    gst_video_overlay_prepare_window_handle (GST_VIDEO_OVERLAY (sink));
    g_mutex_lock (&sink->render_lock);

698
    if (!sink->window) {
699
      /* if we were not provided a window, create one ourselves */
700
      sink->window = gst_wl_window_new_toplevel (sink->display,
701
          &sink->video_info, sink->fullscreen, &sink->render_lock);
702 703
      g_signal_connect_object (sink->window, "closed",
          G_CALLBACK (on_window_closed), sink, 0);
704 705 706
    }
  }

707
  /* drop buffers until we get a frame callback */
708 709
  if (sink->redraw_pending) {
    GST_LOG_OBJECT (sink, "buffer %p dropped (redraw pending)", buffer);
710
    goto done;
711
  }
712

713 714
  /* make sure that the application has called set_render_rectangle() */
  if (G_UNLIKELY (sink->window->render_rectangle.w == 0))
715 716
    goto no_window_size;

717
  wlbuffer = gst_buffer_get_wl_buffer (buffer);
718

719 720 721
  if (G_LIKELY (wlbuffer && wlbuffer->display == sink->display)) {
    GST_LOG_OBJECT (sink, "buffer %p has a wl_buffer from our display, "
        "writing directly", buffer);
Wim Taymans's avatar
Wim Taymans committed
722
    to_render = buffer;
723 724 725 726
    goto render;
  }

  /* update video info from video meta */
727 728
  mem = gst_buffer_peek_memory (buffer, 0);

729
  old_vinfo = sink->video_info;
730 731 732 733 734 735 736
  vmeta = gst_buffer_get_video_meta (buffer);
  if (vmeta) {
    gint i;

    for (i = 0; i < vmeta->n_planes; i++) {
      sink->video_info.offset[i] = vmeta->offset[i];
      sink->video_info.stride[i] = vmeta->stride[i];
737
    }
738
    sink->video_info.size = gst_buffer_get_size (buffer);
739
  }
740

741 742
  GST_LOG_OBJECT (sink, "buffer %p does not have a wl_buffer from our "
      "display, creating it", buffer);
743

744 745 746
  format = GST_VIDEO_INFO_FORMAT (&sink->video_info);
  if (gst_wl_display_check_format_for_dmabuf (sink->display, format)) {
    guint i, nb_dmabuf = 0;
747

748 749 750
    for (i = 0; i < gst_buffer_n_memory (buffer); i++)
      if (gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, i)))
        nb_dmabuf++;
751

752 753 754 755
    if (nb_dmabuf && (nb_dmabuf == gst_buffer_n_memory (buffer)))
      wbuf = gst_wl_linux_dmabuf_construct_wl_buffer (buffer, sink->display,
          &sink->video_info);
  }
756

757
  if (!wbuf && gst_wl_display_check_format_for_shm (sink->display, format)) {
758
    if (gst_buffer_n_memory (buffer) == 1 && gst_is_fd_memory (mem))
759 760
      wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display,
          &sink->video_info);
761 762 763

    /* If nothing worked, copy into our internal pool */
    if (!wbuf) {
764 765 766
      GstVideoFrame src, dst;
      GstVideoInfo src_info = sink->video_info;

767 768 769
      /* rollback video info changes */
      sink->video_info = old_vinfo;

770
      /* we don't know how to create a wl_buffer directly from the provided
771
       * memory, so we have to copy the data to shm memory that we know how
772 773 774 775 776 777 778
       * to handle... */

      GST_LOG_OBJECT (sink, "buffer %p cannot have a wl_buffer, "
          "copying to wl_shm memory", buffer);

      /* sink->pool always exists (created in set_caps), but it may not
       * be active if upstream is not using it */
779 780 781 782 783
      if (!gst_buffer_pool_is_active (sink->pool)) {
        GstStructure *config;
        GstCaps *caps;

        config = gst_buffer_pool_get_config (sink->pool);
784
        gst_buffer_pool_config_get_params (config, &caps, NULL, NULL, NULL);
785 786 787

        /* revert back to default strides and offsets */
        gst_video_info_from_caps (&sink->video_info, caps);
788 789
        gst_buffer_pool_config_set_params (config, caps, sink->video_info.size,
            2, 0);
790 791

        /* This is a video pool, it should not fail with basic setings */
792 793
        if (!gst_buffer_pool_set_config (sink->pool, config) ||
            !gst_buffer_pool_set_active (sink->pool, TRUE))
794 795
          goto activate_failed;
      }
796 797 798 799 800

      ret = gst_buffer_pool_acquire_buffer (sink->pool, &to_render, NULL);
      if (ret != GST_FLOW_OK)
        goto no_buffer;

801
      wlbuffer = gst_buffer_get_wl_buffer (to_render);
802 803

      /* attach a wl_buffer if there isn't one yet */
804 805 806 807
      if (G_UNLIKELY (!wlbuffer)) {
        mem = gst_buffer_peek_memory (to_render, 0);
        wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display,
            &sink->video_info);
808

809
        if (G_UNLIKELY (!wbuf))
810
          goto no_wl_buffer_shm;
811

812
        gst_buffer_add_wl_buffer (to_render, wbuf, sink->display);
813
      }
814

815 816 817 818 819 820 821 822 823 824 825 826 827
      if (!gst_video_frame_map (&dst, &sink->video_info, to_render,
              GST_MAP_WRITE))
        goto dst_map_failed;

      if (!gst_video_frame_map (&src, &src_info, buffer, GST_MAP_READ)) {
        gst_video_frame_unmap (&dst);
        goto src_map_failed;
      }

      gst_video_frame_copy (&dst, &src);

      gst_video_frame_unmap (&src);
      gst_video_frame_unmap (&dst);
828 829

      goto render;
830
    }
Wim Taymans's avatar
Wim Taymans committed
831
  }
832

833 834 835 836 837 838 839
  if (!wbuf)
    goto no_wl_buffer;

  gst_buffer_add_wl_buffer (buffer, wbuf, sink->display);
  to_render = buffer;

render:
840 841 842 843 844 845
  /* drop double rendering */
  if (G_UNLIKELY (to_render == sink->last_buffer)) {
    GST_LOG_OBJECT (sink, "Buffer already being rendered");
    goto done;
  }

846
  gst_buffer_replace (&sink->last_buffer, to_render);
847
  render_last_buffer (sink, FALSE);
848

Wim Taymans's avatar
Wim Taymans committed
849 850
  if (buffer != to_render)
    gst_buffer_unref (to_render);
851
  goto done;
Wim Taymans's avatar
Wim Taymans committed
852

853 854 855 856 857 858 859 860
no_window_size:
  {
    GST_ELEMENT_ERROR (sink, RESOURCE, WRITE,
        ("Window has no size set"),
        ("Make sure you set the size after calling set_window_handle"));
    ret = GST_FLOW_ERROR;
    goto done;
  }
861 862
no_buffer:
  {
863
    GST_WARNING_OBJECT (sink, "could not create buffer");
864
    goto done;
865
  }
866
no_wl_buffer_shm:
Wim Taymans's avatar
Wim Taymans committed
867
  {
868
    GST_ERROR_OBJECT (sink, "could not create wl_buffer out of wl_shm memory");
869 870
    ret = GST_FLOW_ERROR;
    goto done;
871
  }
872 873 874 875 876 877
no_wl_buffer:
  {
    GST_ERROR_OBJECT (sink, "buffer %p cannot have a wl_buffer", buffer);
    ret = GST_FLOW_ERROR;
    goto done;
  }
878 879 880 881
activate_failed:
  {
    GST_ERROR_OBJECT (sink, "failed to activate bufferpool.");
    ret = GST_FLOW_ERROR;
882 883
    goto done;
  }
884 885 886 887 888 889 890 891 892 893 894 895 896 897
src_map_failed:
  {
    GST_ELEMENT_ERROR (sink, RESOURCE, READ,
        ("Video memory can not be read from userspace."), (NULL));
    ret = GST_FLOW_ERROR;
    goto done;
  }
dst_map_failed:
  {
    GST_ELEMENT_ERROR (sink, RESOURCE, WRITE,
        ("Video memory can not be written from userspace."), (NULL));
    ret = GST_FLOW_ERROR;
    goto done;
  }
898 899
done:
  {
900
    g_mutex_unlock (&sink->render_lock);
901
    return ret;
Wim Taymans's avatar
Wim Taymans committed
902
  }
903 904
}

905 906 907 908
static void
gst_wayland_sink_videooverlay_init (GstVideoOverlayInterface * iface)
{
  iface->set_window_handle = gst_wayland_sink_set_window_handle;
909
  iface->set_render_rectangle = gst_wayland_sink_set_render_rectangle;
910 911 912 913 914 915 916
  iface->expose = gst_wayland_sink_expose;
}

static void
gst_wayland_sink_set_window_handle (GstVideoOverlay * overlay, guintptr handle)
{
  GstWaylandSink *sink = GST_WAYLAND_SINK (overlay);
917
  struct wl_surface *surface = (struct wl_surface *) handle;
918

919
  g_return_if_fail (sink != NULL);
920

921 922 923 924 925
  if (sink->window != NULL) {
    GST_WARNING_OBJECT (sink, "changing window handle is not supported");
    return;
  }

926
  g_mutex_lock (&sink->render_lock);
927 928 929 930 931 932 933

  GST_DEBUG_OBJECT (sink, "Setting window handle %" GST_PTR_FORMAT,
      (void *) handle);

  g_clear_object (&sink->window);

  if (handle) {
934 935 936 937 938 939 940 941 942
    if (G_LIKELY (gst_wayland_sink_find_display (sink))) {
      /* we cannot use our own display with an external window handle */
      if (G_UNLIKELY (sink->display->own_display)) {
        GST_ELEMENT_ERROR (sink, RESOURCE, OPEN_READ_WRITE,
            ("Application did not provide a wayland display handle"),
            ("waylandsink cannot use an externally-supplied surface without "
                "an externally-supplied display handle. Consider providing a "
                "display handle from your application with GstContext"));
      } else {
943 944
        sink->window = gst_wl_window_new_in_surface (sink->display, surface,
            &sink->render_lock);
945
      }
946
    } else {
947 948
      GST_ERROR_OBJECT (sink, "Failed to find display handle, "
          "ignoring window handle");
949 950 951
    }
  }

952
  g_mutex_unlock (&sink->render_lock);
953 954
}

955 956 957 958 959 960 961 962
static void
gst_wayland_sink_set_render_rectangle (GstVideoOverlay * overlay,
    gint x, gint y, gint w, gint h)
{
  GstWaylandSink *sink = GST_WAYLAND_SINK (overlay);

  g_return_if_fail (sink != NULL);

963
  g_mutex_lock (&sink->render_lock);
964
  if (!sink->window) {
965
    g_mutex_unlock (&sink->render_lock);
966 967 968 969 970 971 972
    GST_WARNING_OBJECT (sink,
        "set_render_rectangle called without window, ignoring");
    return;
  }

  GST_DEBUG_OBJECT (sink, "window geometry changed to (%d, %d) %d x %d",
      x, y, w, h);
973 974
  gst_wl_window_set_render_rectangle (sink->window, x, y, w, h);

975
  g_mutex_unlock (&sink->render_lock);
976 977
}

978 979 980 981
static void
gst_wayland_sink_expose (GstVideoOverlay * overlay)
{
  GstWaylandSink *sink = GST_WAYLAND_SINK (overlay);
982

983
  g_return_if_fail (sink != NULL);
984 985 986

  GST_DEBUG_OBJECT (sink, "expose");

987
  g_mutex_lock (&sink->render_lock);
988
  if (sink->last_buffer && !sink->redraw_pending) {
989
    GST_DEBUG_OBJECT (sink, "redrawing last buffer");
990
    render_last_buffer (sink, TRUE);
991
  }
992
  g_mutex_unlock (&sink->render_lock);
993 994 995 996 997
}

static void
gst_wayland_sink_waylandvideo_init (GstWaylandVideoInterface * iface)
{
998 999
  iface->begin_geometry_change = gst_wayland_sink_begin_geometry_change;
  iface->end_geometry_change = gst_wayland_sink_end_geometry_change;
1000 1001 1002
}

static void
1003
gst_wayland_sink_begin_geometry_change (GstWaylandVideo * video)
1004 1005 1006
{
  GstWaylandSink *sink = GST_WAYLAND_SINK (video);
  g_return_if_fail (sink != NULL);
1007

1008
  g_mutex_lock (&sink->render_lock);
1009
  if (!sink->window || !sink->window->area_subsurface) {
1010
    g_mutex_unlock (&sink->render_lock);
1011 1012
    GST_INFO_OBJECT (sink,
        "begin_geometry_change called without window, ignoring");
1013 1014 1015
    return;
  }

1016
  wl_subsurface_set_sync (sink->window->area_subsurface);
1017
  g_mutex_unlock (&sink->render_lock);
1018 1019 1020
}

static void
1021
gst_wayland_sink_end_geometry_change (GstWaylandVideo * video)
1022 1023 1024
{
  GstWaylandSink *sink = GST_WAYLAND_SINK (video);
  g_return_if_fail (sink != NULL);
1025

1026
  g_mutex_lock (&sink->render_lock);
1027
  if (!sink->window || !sink->window->area_subsurface) {
1028
    g_mutex_unlock (&sink->render_lock);
1029 1030
    GST_INFO_OBJECT (sink,
        "end_geometry_change called without window, ignoring");
1031
    return;
1032
  }
1033

1034
  wl_subsurface_set_desync (sink->window->area_subsurface);
1035
  g_mutex_unlock (&sink->render_lock);
1036 1037
}

1038 1039 1040 1041 1042 1043
static gboolean
plugin_init (GstPlugin * plugin)
{
  GST_DEBUG_CATEGORY_INIT (gstwayland_debug, "waylandsink", 0,
      " wayland video sink");

1044 1045
  gst_wl_shm_allocator_register ();

1046 1047 1048 1049 1050 1051
  return gst_element_register (plugin, "waylandsink", GST_RANK_MARGINAL,
      GST_TYPE_WAYLAND_SINK);
}

GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
    GST_VERSION_MINOR,
1052
    waylandsink,
1053 1054
    "Wayland Video Sink", plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME,
    GST_PACKAGE_ORIGIN)