videomixer.c 55.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/* Generic video mixer plugin
 * Copyright (C) 2004 Wim Taymans <wim@fluendo.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 */

20
21
22
/**
 * SECTION:element-videomixer
 *
23
 * Videomixer can accept AYUV, ARGB and BGRA video streams. For each of the requested
24
25
26
 * sink pads it will compare the incoming geometry and framerate to define the
 * output parameters. Indeed output video frames will have the geometry of the
 * biggest incoming video stream and the framerate of the fastest incoming one.
27
 *
28
 * All sink pads must be either AYUV, ARGB or BGRA, but a mixture of them is not 
29
30
31
32
 * supported. The src pad will have the same colorspace as the sinks. 
 * No colorspace conversion is done. 
 * 
 *
33
34
 * Individual parameters for each input stream can be configured on the
 * #GstVideoMixerPad.
35
36
 *
 * <refsect2>
37
 * <title>Sample pipelines</title>
38
 * |[
39
 * gst-launch videotestsrc pattern=1 ! video/x-raw-yuv,format=\(fourcc\)AYUV, framerate=\(fraction\)10/1, width=100, height=100 ! videobox border-alpha=0 alpha=0.5 top=-70 bottom=-70 right=-220 ! videomixer name=mix ! ffmpegcolorspace ! xvimagesink videotestsrc ! video/x-raw-yuv, format=\(fourcc\)AYUV, framerate=\(fraction\)5/1, width=320, height=240 ! alpha alpha=0.7 ! mix.
40
 * ]| A pipeline to demonstrate videomixer used together with videobox.
41
42
43
44
45
46
 * This should show a 320x240 pixels video test source with some transparency
 * showing the background checker pattern. Another video test source with just
 * the snow pattern of 100x100 pixels is overlayed on top of the first one on
 * the left vertically centered with a small transparency showing the first
 * video test source behind and the checker pattern under it. Note that the
 * framerate of the output video is 10 frames per second.
47
48
49
50
51
52
 * |[
 * gst-launch videotestsrc pattern=1 ! video/x-raw-rgb, framerate=\(fraction\)10/1, width=100, height=100 ! videomixer name=mix ! ffmpegcolorspace ! ximagesink videotestsrc ! video/x-raw-rgb, framerate=\(fraction\)5/1, width=320, height=240 ! mix.
 * ]| A pipeline to demostrate bgra mixing. (This does not demonstrate alpha blending). 
 * |[
 * gst-launch   videotestsrc pattern=1 ! video/x-raw-yuv,format =\(fourcc\)I420, framerate=\(fraction\)10/1, width=100, height=100 ! videomixer name=mix ! ffmpegcolorspace ! ximagesink videotestsrc ! video/x-raw-yuv,format=\(fourcc\)I420, framerate=\(fraction\)5/1, width=320, height=240 ! mix.
 * ]| A pipeline to test I420
53
54
55
 * </refsect2>
 */

56
57
58
59
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

60
61
62
63
64
65
#ifdef HAVE_GCC_ASM
#if defined(HAVE_CPU_I386) || defined(HAVE_CPU_X86_64)
#define BUILD_X86_ASM
#endif
#endif

66
#include <gst/gst.h>
67
#include <gst/base/gstcollectpads.h>
68
#include <gst/controller/gstcontroller.h>
69
#include <gst/video/video.h>
70

71
72
73
74
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef HAVE_STRING_H
75
#include <string.h>
76
#endif
77

78
79
#include "videomixer.h"

80
81
GST_DEBUG_CATEGORY_STATIC (gst_videomixer_debug);
#define GST_CAT_DEFAULT gst_videomixer_debug
82

83
84
85
86
87
88
89
#define GST_VIDEO_MIXER_GET_STATE_LOCK(mix) \
  (GST_VIDEO_MIXER(mix)->state_lock)
#define GST_VIDEO_MIXER_STATE_LOCK(mix) \
  (g_mutex_lock(GST_VIDEO_MIXER_GET_STATE_LOCK (mix)))
#define GST_VIDEO_MIXER_STATE_UNLOCK(mix) \
  (g_mutex_unlock(GST_VIDEO_MIXER_GET_STATE_LOCK (mix)))

90
91
static GType gst_videomixer_get_type (void);

92
93
94
95
96
static void gst_videomixer_pad_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);
static void gst_videomixer_pad_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);

97
98
99
static gboolean gst_videomixer_src_event (GstPad * pad, GstEvent * event);
static gboolean gst_videomixer_sink_event (GstPad * pad, GstEvent * event);

100
101
static void gst_videomixer_sort_pads (GstVideoMixer * mix);

102
103
104
105
106
107
#define DEFAULT_PAD_ZORDER 0
#define DEFAULT_PAD_XPOS   0
#define DEFAULT_PAD_YPOS   0
#define DEFAULT_PAD_ALPHA  1.0
enum
{
Sebastian Dröge's avatar
Sebastian Dröge committed
108
109
110
111
112
  PROP_PAD_0,
  PROP_PAD_ZORDER,
  PROP_PAD_XPOS,
  PROP_PAD_YPOS,
  PROP_PAD_ALPHA
113
114
};

115
GType gst_videomixer_pad_get_type (void);
Sebastian Dröge's avatar
Sebastian Dröge committed
116
G_DEFINE_TYPE (GstVideoMixerPad, gst_videomixer_pad, GST_TYPE_PAD);
117
118
119
120

static void
gst_videomixer_pad_class_init (GstVideoMixerPadClass * klass)
{
Sebastian Dröge's avatar
Sebastian Dröge committed
121
  GObjectClass *gobject_class = (GObjectClass *) klass;
122

Sebastian Dröge's avatar
Sebastian Dröge committed
123
124
  gobject_class->set_property = gst_videomixer_pad_set_property;
  gobject_class->get_property = gst_videomixer_pad_get_property;
125

Sebastian Dröge's avatar
Sebastian Dröge committed
126
  g_object_class_install_property (gobject_class, PROP_PAD_ZORDER,
127
      g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture",
128
          0, 10000, DEFAULT_PAD_ZORDER,
Sebastian Dröge's avatar
Sebastian Dröge committed
129
130
          G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_PAD_XPOS,
131
      g_param_spec_int ("xpos", "X Position", "X Position of the picture",
132
          G_MININT, G_MAXINT, DEFAULT_PAD_XPOS,
Sebastian Dröge's avatar
Sebastian Dröge committed
133
134
          G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_PAD_YPOS,
135
      g_param_spec_int ("ypos", "Y Position", "Y Position of the picture",
136
          G_MININT, G_MAXINT, DEFAULT_PAD_YPOS,
Sebastian Dröge's avatar
Sebastian Dröge committed
137
138
          G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_PAD_ALPHA,
139
      g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0,
Sebastian Dröge's avatar
Sebastian Dröge committed
140
141
          DEFAULT_PAD_ALPHA,
          G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
142
143
144
145
146
147
}

static void
gst_videomixer_pad_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
148
  GstVideoMixerPad *pad = GST_VIDEO_MIXER_PAD (object);
149
150

  switch (prop_id) {
Sebastian Dröge's avatar
Sebastian Dröge committed
151
    case PROP_PAD_ZORDER:
152
153
      g_value_set_uint (value, pad->zorder);
      break;
Sebastian Dröge's avatar
Sebastian Dröge committed
154
    case PROP_PAD_XPOS:
155
156
      g_value_set_int (value, pad->xpos);
      break;
Sebastian Dröge's avatar
Sebastian Dröge committed
157
    case PROP_PAD_YPOS:
158
159
      g_value_set_int (value, pad->ypos);
      break;
Sebastian Dröge's avatar
Sebastian Dröge committed
160
    case PROP_PAD_ALPHA:
161
162
163
164
165
166
167
168
169
170
171
172
      g_value_set_double (value, pad->alpha);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_videomixer_pad_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
Sebastian Dröge's avatar
Sebastian Dröge committed
173
174
  GstVideoMixerPad *pad = GST_VIDEO_MIXER_PAD (object);
  GstVideoMixer *mix = GST_VIDEO_MIXER (gst_pad_get_parent (GST_PAD (pad)));
175
176

  switch (prop_id) {
Sebastian Dröge's avatar
Sebastian Dröge committed
177
    case PROP_PAD_ZORDER:
178
      GST_VIDEO_MIXER_STATE_LOCK (mix);
179
      pad->zorder = g_value_get_uint (value);
180
      gst_videomixer_sort_pads (mix);
181
      GST_VIDEO_MIXER_STATE_UNLOCK (mix);
182
      break;
Sebastian Dröge's avatar
Sebastian Dröge committed
183
    case PROP_PAD_XPOS:
184
185
      pad->xpos = g_value_get_int (value);
      break;
Sebastian Dröge's avatar
Sebastian Dröge committed
186
    case PROP_PAD_YPOS:
187
188
      pad->ypos = g_value_get_int (value);
      break;
Sebastian Dröge's avatar
Sebastian Dröge committed
189
    case PROP_PAD_ALPHA:
190
191
192
193
194
195
      pad->alpha = g_value_get_double (value);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
196
197

  gst_object_unref (mix);
198
199
}

200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
static void
gst_videomixer_update_qos (GstVideoMixer * mix, gdouble proportion,
    GstClockTimeDiff diff, GstClockTime timestamp)
{
  GST_DEBUG_OBJECT (mix,
      "Updating QoS: proportion %lf, diff %s%" GST_TIME_FORMAT ", timestamp %"
      GST_TIME_FORMAT, proportion, (diff < 0) ? "-" : "",
      GST_TIME_ARGS (ABS (diff)), GST_TIME_ARGS (timestamp));

  GST_OBJECT_LOCK (mix);
  mix->proportion = proportion;
  if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) {
    if (G_UNLIKELY (diff > 0))
      mix->earliest_time =
          timestamp + 2 * diff + gst_util_uint64_scale_int (GST_SECOND,
          mix->fps_d, mix->fps_n);
    else
      mix->earliest_time = timestamp + diff;
  } else {
    mix->earliest_time = GST_CLOCK_TIME_NONE;
  }
  GST_OBJECT_UNLOCK (mix);
}

static void
gst_videomixer_reset_qos (GstVideoMixer * mix)
{
  gst_videomixer_update_qos (mix, 0.5, 0, GST_CLOCK_TIME_NONE);
}

static void
gst_videomixer_read_qos (GstVideoMixer * mix, gdouble * proportion,
    GstClockTime * time)
{
  GST_OBJECT_LOCK (mix);
  *proportion = mix->proportion;
  *time = mix->earliest_time;
  GST_OBJECT_UNLOCK (mix);
}

/* Perform qos calculations before processing the next frame. Returns TRUE if
 * the frame should be processed, FALSE if the frame can be dropped entirely */
static gboolean
gst_videomixer_do_qos (GstVideoMixer * mix, GstClockTime timestamp)
{
  GstClockTime qostime, earliest_time;
  gdouble proportion;

  /* no timestamp, can't do QoS => process frame */
  if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (timestamp))) {
    GST_LOG_OBJECT (mix, "invalid timestamp, can't do QoS, process frame");
    return TRUE;
  }

  /* get latest QoS observation values */
  gst_videomixer_read_qos (mix, &proportion, &earliest_time);

  /* skip qos if we have no observation (yet) => process frame */
  if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (earliest_time))) {
    GST_LOG_OBJECT (mix, "no observation yet, process frame");
    return TRUE;
  }

  /* qos is done on running time */
  qostime =
      gst_segment_to_running_time (&mix->segment, GST_FORMAT_TIME, timestamp);

  /* see how our next timestamp relates to the latest qos timestamp */
  GST_LOG_OBJECT (mix, "qostime %" GST_TIME_FORMAT ", earliest %"
      GST_TIME_FORMAT, GST_TIME_ARGS (qostime), GST_TIME_ARGS (earliest_time));

  if (qostime != GST_CLOCK_TIME_NONE && qostime <= earliest_time) {
    GST_DEBUG_OBJECT (mix, "we are late, drop frame");
    return FALSE;
  }

  GST_LOG_OBJECT (mix, "process frame");
  return TRUE;
}

280
281
282
283
static void
gst_videomixer_set_master_geometry (GstVideoMixer * mix)
{
  GSList *walk;
284
  gint width = 0, height = 0, fps_n = 0, fps_d = 0, par_n = 0, par_d = 0;
285
286
287
288
289
290
291
292
293
294
295
296
297
  GstVideoMixerPad *master = NULL;

  walk = mix->sinkpads;
  while (walk) {
    GstVideoMixerPad *mixpad = GST_VIDEO_MIXER_PAD (walk->data);

    walk = g_slist_next (walk);

    /* Biggest input geometry will be our output geometry */
    width = MAX (width, mixpad->in_width);
    height = MAX (height, mixpad->in_height);

    /* If mix framerate < mixpad framerate, using fractions */
298
    GST_DEBUG_OBJECT (mixpad, "comparing framerate %d/%d to mixpad's %d/%d",
299
300
301
302
303
        fps_n, fps_d, mixpad->fps_n, mixpad->fps_d);
    if ((!fps_n && !fps_d) ||
        ((gint64) fps_n * mixpad->fps_d < (gint64) mixpad->fps_n * fps_d)) {
      fps_n = mixpad->fps_n;
      fps_d = mixpad->fps_d;
304
305
      par_n = mixpad->par_n;
      par_d = mixpad->par_d;
306
307
308
309
310
311
312
313
      GST_DEBUG_OBJECT (mixpad, "becomes the master pad");
      master = mixpad;
    }
  }

  /* set results */
  if (mix->master != master || mix->in_width != width
      || mix->in_height != height || mix->fps_n != fps_n
314
      || mix->fps_d != fps_d || mix->par_n != par_n || mix->par_d != par_d) {
315
    mix->setcaps = TRUE;
316
    mix->sendseg = TRUE;
317
    gst_videomixer_reset_qos (mix);
318
319
320
321
322
    mix->master = master;
    mix->in_width = width;
    mix->in_height = height;
    mix->fps_n = fps_n;
    mix->fps_d = fps_d;
323
324
    mix->par_n = par_n;
    mix->par_d = par_d;
325
326
327
  }
}

328
329
static gboolean
gst_videomixer_pad_sink_setcaps (GstPad * pad, GstCaps * vscaps)
330
331
332
333
{
  GstVideoMixer *mix;
  GstVideoMixerPad *mixpad;
  GstStructure *structure;
334
  gint in_width, in_height;
335
  gboolean ret = FALSE;
336
  const GValue *framerate, *par;
337
338

  GST_INFO_OBJECT (pad, "Setting caps %" GST_PTR_FORMAT, vscaps);
339
340
341
342

  mix = GST_VIDEO_MIXER (gst_pad_get_parent (pad));
  mixpad = GST_VIDEO_MIXER_PAD (pad);

343
  if (!mixpad)
344
345
    goto beach;

346
347
  structure = gst_caps_get_structure (vscaps, 0);

348
349
  if (!gst_structure_get_int (structure, "width", &in_width)
      || !gst_structure_get_int (structure, "height", &in_height)
350
      || (framerate = gst_structure_get_value (structure, "framerate")) == NULL)
351
    goto beach;
352
  par = gst_structure_get_value (structure, "pixel-aspect-ratio");
353

354
  GST_VIDEO_MIXER_STATE_LOCK (mix);
355
356
  mixpad->fps_n = gst_value_get_fraction_numerator (framerate);
  mixpad->fps_d = gst_value_get_fraction_denominator (framerate);
357
358
359
360
361
362
  if (par) {
    mixpad->par_n = gst_value_get_fraction_numerator (par);
    mixpad->par_d = gst_value_get_fraction_denominator (par);
  } else {
    mixpad->par_n = mixpad->par_d = 1;
  }
363

364
365
  mixpad->in_width = in_width;
  mixpad->in_height = in_height;
366

367
  gst_videomixer_set_master_geometry (mix);
368
  GST_VIDEO_MIXER_STATE_UNLOCK (mix);
369

370
371
372
373
374
375
  ret = TRUE;

beach:
  gst_object_unref (mix);

  return ret;
376
377
}

378
379
380
381
382
383
static GstCaps *
gst_videomixer_pad_sink_getcaps (GstPad * pad)
{
  GstVideoMixer *mix;
  GstVideoMixerPad *mixpad;
  GstCaps *res = NULL;
384
  GstCaps *mastercaps;
385
386
387
388
389
390
391
392
  GstStructure *st;

  mix = GST_VIDEO_MIXER (gst_pad_get_parent (pad));
  mixpad = GST_VIDEO_MIXER_PAD (pad);

  if (!mixpad)
    goto beach;

393
394
  /* Get downstream allowed caps */
  res = gst_pad_get_allowed_caps (mix->srcpad);
395
396
397
398
  if (G_UNLIKELY (res == NULL)) {
    res = gst_caps_copy (gst_pad_get_pad_template_caps (pad));
    goto beach;
  }
399
400
401
402
403
404

  GST_VIDEO_MIXER_STATE_LOCK (mix);

  /* Return as-is if not other sinkpad set as master */
  if (mix->master == NULL) {
    GST_VIDEO_MIXER_STATE_UNLOCK (mix);
405
406
407
    goto beach;
  }

408
  mastercaps = gst_pad_get_fixed_caps_func (GST_PAD (mix->master));
409

410
411
412
413
414
415
416
  /* If master pad caps aren't negotiated yet, return downstream
   * allowed caps */
  if (!GST_CAPS_IS_SIMPLE (mastercaps)) {
    GST_VIDEO_MIXER_STATE_UNLOCK (mix);
    gst_caps_unref (mastercaps);
    goto beach;
  }
417

418
419
420
421
422
423
424
425
  gst_caps_unref (res);
  res = gst_caps_make_writable (mastercaps);
  st = gst_caps_get_structure (res, 0);
  gst_structure_set (st, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
      "height", GST_TYPE_INT_RANGE, 1, G_MAXINT,
      "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL);
  if (!gst_structure_has_field (st, "pixel-aspect-ratio"))
    gst_structure_set (st, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL);
426
427
428
429
430
431
432
433
434
435

  GST_VIDEO_MIXER_STATE_UNLOCK (mix);


beach:
  GST_DEBUG_OBJECT (pad, "Returning %" GST_PTR_FORMAT, res);

  return res;
}

436
/*
437
438
439
440
441
442
443
444
445
* We accept the caps if it has the same format as other sink pads in 
* the element.
*/
static gboolean
gst_videomixer_pad_sink_acceptcaps (GstPad * pad, GstCaps * vscaps)
{
  gboolean ret;
  GstVideoMixer *mix;
  GstCaps *acceptedCaps;
446

447
  mix = GST_VIDEO_MIXER (gst_pad_get_parent (pad));
448
  GST_DEBUG_OBJECT (pad, "%" GST_PTR_FORMAT, vscaps);
449
450
451
452
453
  GST_VIDEO_MIXER_STATE_LOCK (mix);

  if (mix->master) {
    acceptedCaps = gst_pad_get_fixed_caps_func (GST_PAD (mix->master));
    acceptedCaps = gst_caps_make_writable (acceptedCaps);
454
    GST_LOG_OBJECT (pad, "master's caps %" GST_PTR_FORMAT, acceptedCaps);
455
    if (GST_CAPS_IS_SIMPLE (acceptedCaps)) {
456
457
458
459
460
461
462
463
      GstStructure *s;
      s = gst_caps_get_structure (acceptedCaps, 0);
      gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
          "height", GST_TYPE_INT_RANGE, 1, G_MAXINT,
          "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL);
      if (!gst_structure_has_field (s, "pixel-aspect-ratio"))
        gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
            NULL);
464
465
466
467
    }
  } else {
    acceptedCaps = gst_pad_get_fixed_caps_func (pad);
  }
468
469
470

  GST_INFO_OBJECT (pad, "vscaps: %" GST_PTR_FORMAT, vscaps);
  GST_INFO_OBJECT (pad, "acceptedCaps: %" GST_PTR_FORMAT, acceptedCaps);
471

472
473
474
  ret = gst_caps_can_intersect (vscaps, acceptedCaps);
  GST_INFO_OBJECT (pad, "%saccepted caps %" GST_PTR_FORMAT, (ret ? "" : "not "),
      vscaps);
475
476
477
478
479
480
  gst_caps_unref (acceptedCaps);
  GST_VIDEO_MIXER_STATE_UNLOCK (mix);
  gst_object_unref (mix);
  return ret;
}

481
482
483
484
485
486


static void
gst_videomixer_pad_init (GstVideoMixerPad * mixerpad)
{
  /* setup some pad functions */
487
488
  gst_pad_set_setcaps_function (GST_PAD (mixerpad),
      gst_videomixer_pad_sink_setcaps);
489
490
  gst_pad_set_acceptcaps_function (GST_PAD (mixerpad),
      GST_DEBUG_FUNCPTR (gst_videomixer_pad_sink_acceptcaps));
491
492
  gst_pad_set_getcaps_function (GST_PAD (mixerpad),
      gst_videomixer_pad_sink_getcaps);
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509

  mixerpad->zorder = DEFAULT_PAD_ZORDER;
  mixerpad->xpos = DEFAULT_PAD_XPOS;
  mixerpad->ypos = DEFAULT_PAD_YPOS;
  mixerpad->alpha = DEFAULT_PAD_ALPHA;
}

/* VideoMixer signals and args */
enum
{
  /* FILL ME */
  LAST_SIGNAL
};

#define DEFAULT_BACKGROUND VIDEO_MIXER_BACKGROUND_CHECKER
enum
{
Sebastian Dröge's avatar
Sebastian Dröge committed
510
511
  PROP_0,
  PROP_BACKGROUND
512
513
514
515
516
517
518
};

#define GST_TYPE_VIDEO_MIXER_BACKGROUND (gst_video_mixer_background_get_type())
static GType
gst_video_mixer_background_get_type (void)
{
  static GType video_mixer_background_type = 0;
519

520
  static const GEnumValue video_mixer_background[] = {
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
521
522
523
    {VIDEO_MIXER_BACKGROUND_CHECKER, "Checker pattern", "checker"},
    {VIDEO_MIXER_BACKGROUND_BLACK, "Black", "black"},
    {VIDEO_MIXER_BACKGROUND_WHITE, "White", "white"},
524
525
526
527
528
529
530
531
532
533
534
535
536
537
    {0, NULL, NULL},
  };

  if (!video_mixer_background_type) {
    video_mixer_background_type =
        g_enum_register_static ("GstVideoMixerBackground",
        video_mixer_background);
  }
  return video_mixer_background_type;
}

static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
538
    GST_STATIC_CAPS (GST_VIDEO_CAPS_YUV ("AYUV") ";" GST_VIDEO_CAPS_BGRA ";"
539
        GST_VIDEO_CAPS_ARGB ";" GST_VIDEO_CAPS_RGBA ";" GST_VIDEO_CAPS_ABGR ";"
540
        GST_VIDEO_CAPS_YUV ("Y444") ";" GST_VIDEO_CAPS_YUV ("Y42B") ";"
541
542
543
        GST_VIDEO_CAPS_YUV ("YUY2") ";" GST_VIDEO_CAPS_YUV ("UYVY") ";"
        GST_VIDEO_CAPS_YUV ("YVYU") ";"
        GST_VIDEO_CAPS_YUV ("I420") ";" GST_VIDEO_CAPS_YUV ("YV12") ";"
544
        GST_VIDEO_CAPS_YUV ("Y41B") ";" GST_VIDEO_CAPS_RGB ";"
545
546
        GST_VIDEO_CAPS_BGR ";" GST_VIDEO_CAPS_xRGB ";" GST_VIDEO_CAPS_xBGR ";"
        GST_VIDEO_CAPS_RGBx ";" GST_VIDEO_CAPS_BGRx)
547
548
549
550
551
    );

static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%d",
    GST_PAD_SINK,
    GST_PAD_REQUEST,
552
    GST_STATIC_CAPS (GST_VIDEO_CAPS_YUV ("AYUV") ";" GST_VIDEO_CAPS_BGRA ";"
553
        GST_VIDEO_CAPS_ARGB ";" GST_VIDEO_CAPS_RGBA ";" GST_VIDEO_CAPS_ABGR ";"
554
        GST_VIDEO_CAPS_YUV ("Y444") ";" GST_VIDEO_CAPS_YUV ("Y42B") ";"
555
556
557
        GST_VIDEO_CAPS_YUV ("YUY2") ";" GST_VIDEO_CAPS_YUV ("UYVY") ";"
        GST_VIDEO_CAPS_YUV ("YVYU") ";"
        GST_VIDEO_CAPS_YUV ("I420") ";" GST_VIDEO_CAPS_YUV ("YV12") ";"
558
        GST_VIDEO_CAPS_YUV ("Y41B") ";" GST_VIDEO_CAPS_RGB ";"
559
560
        GST_VIDEO_CAPS_BGR ";" GST_VIDEO_CAPS_xRGB ";" GST_VIDEO_CAPS_xBGR ";"
        GST_VIDEO_CAPS_RGBx ";" GST_VIDEO_CAPS_BGRx)
561
562
    );

563
static void gst_videomixer_finalize (GObject * object);
564

565
static GstCaps *gst_videomixer_getcaps (GstPad * pad);
566
static gboolean gst_videomixer_setcaps (GstPad * pad, GstCaps * caps);
567
static gboolean gst_videomixer_query (GstPad * pad, GstQuery * query);
568

569
570
static GstFlowReturn gst_videomixer_collected (GstCollectPads * pads,
    GstVideoMixer * mix);
571
572
static GstPad *gst_videomixer_request_new_pad (GstElement * element,
    GstPadTemplate * templ, const gchar * name);
573
static void gst_videomixer_release_pad (GstElement * element, GstPad * pad);
574

575
576
577
578
static void gst_videomixer_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_videomixer_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);
579
580
static GstStateChangeReturn gst_videomixer_change_state (GstElement * element,
    GstStateChange transition);
581
582
583

/*static guint gst_videomixer_signals[LAST_SIGNAL] = { 0 }; */

584
585
586
587
588
589
590
591
592
static void gst_videomixer_child_proxy_init (gpointer g_iface,
    gpointer iface_data);
static void _do_init (GType object_type);

GST_BOILERPLATE_FULL (GstVideoMixer, gst_videomixer, GstElement,
    GST_TYPE_ELEMENT, _do_init);

static void
_do_init (GType object_type)
593
{
Sebastian Dröge's avatar
Sebastian Dröge committed
594
  static const GInterfaceInfo child_proxy_info = {
595
596
597
598
    (GInterfaceInitFunc) gst_videomixer_child_proxy_init,
    NULL,
    NULL
  };
Sebastian Dröge's avatar
Sebastian Dröge committed
599

600
601
602
603
  g_type_add_interface_static (object_type, GST_TYPE_CHILD_PROXY,
      &child_proxy_info);
  GST_INFO ("GstChildProxy interface registered");
}
604

605
606
607
608
609
610
static GstObject *
gst_videomixer_child_proxy_get_child_by_index (GstChildProxy * child_proxy,
    guint index)
{
  GstVideoMixer *mix = GST_VIDEO_MIXER (child_proxy);
  GstObject *obj;
611

612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
  GST_VIDEO_MIXER_STATE_LOCK (mix);
  if ((obj = g_slist_nth_data (mix->sinkpads, index)))
    gst_object_ref (obj);
  GST_VIDEO_MIXER_STATE_UNLOCK (mix);
  return obj;
}

static guint
gst_videomixer_child_proxy_get_children_count (GstChildProxy * child_proxy)
{
  guint count = 0;
  GstVideoMixer *mix = GST_VIDEO_MIXER (child_proxy);

  GST_VIDEO_MIXER_STATE_LOCK (mix);
  count = mix->numpads;
  GST_VIDEO_MIXER_STATE_UNLOCK (mix);
628
  GST_INFO_OBJECT (mix, "Children Count: %d", count);
629
630
631
632
633
634
635
636
637
638
639
  return count;
}

static void
gst_videomixer_child_proxy_init (gpointer g_iface, gpointer iface_data)
{
  GstChildProxyInterface *iface = g_iface;

  GST_INFO ("intializing child proxy interface");
  iface->get_child_by_index = gst_videomixer_child_proxy_get_child_by_index;
  iface->get_children_count = gst_videomixer_child_proxy_get_children_count;
640
641
642
643
644
645
646
647
648
649
650
651
}

static void
gst_videomixer_base_init (gpointer g_class)
{
  GstElementClass *element_class = GST_ELEMENT_CLASS (g_class);

  gst_element_class_add_pad_template (element_class,
      gst_static_pad_template_get (&src_factory));
  gst_element_class_add_pad_template (element_class,
      gst_static_pad_template_get (&sink_factory));

Sebastian Dröge's avatar
Sebastian Dröge committed
652
653
654
  gst_element_class_set_details_simple (element_class, "Video mixer",
      "Filter/Editor/Video",
      "Mix multiple video streams", "Wim Taymans <wim@fluendo.com>");
655
656
657
658
659
}

static void
gst_videomixer_class_init (GstVideoMixerClass * klass)
{
Sebastian Dröge's avatar
Sebastian Dröge committed
660
661
  GObjectClass *gobject_class = (GObjectClass *) klass;
  GstElementClass *gstelement_class = (GstElementClass *) klass;
662

663
  gobject_class->finalize = gst_videomixer_finalize;
664

665
666
667
  gobject_class->get_property = gst_videomixer_get_property;
  gobject_class->set_property = gst_videomixer_set_property;

Sebastian Dröge's avatar
Sebastian Dröge committed
668
  g_object_class_install_property (gobject_class, PROP_BACKGROUND,
669
670
      g_param_spec_enum ("background", "Background", "Background type",
          GST_TYPE_VIDEO_MIXER_BACKGROUND,
Sebastian Dröge's avatar
Sebastian Dröge committed
671
          DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
672

673
674
675
676
677
678
  gstelement_class->request_new_pad =
      GST_DEBUG_FUNCPTR (gst_videomixer_request_new_pad);
  gstelement_class->release_pad =
      GST_DEBUG_FUNCPTR (gst_videomixer_release_pad);
  gstelement_class->change_state =
      GST_DEBUG_FUNCPTR (gst_videomixer_change_state);
679
680
681

  /* Register the pad class */
  (void) (GST_TYPE_VIDEO_MIXER_PAD);
Sebastian Dröge's avatar
Sebastian Dröge committed
682
683
  /* Register the background enum */
  (void) (GST_TYPE_VIDEO_MIXER_BACKGROUND);
684
685
686
}

static void
687
gst_videomixer_collect_free (GstVideoMixerCollect * mixcol)
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
{
  if (mixcol->buffer) {
    gst_buffer_unref (mixcol->buffer);
    mixcol->buffer = NULL;
  }
}

static void
gst_videomixer_reset (GstVideoMixer * mix)
{
  GSList *walk;

  mix->in_width = 0;
  mix->in_height = 0;
  mix->out_width = 0;
  mix->out_height = 0;
  mix->fps_n = mix->fps_d = 0;
705
  mix->par_n = mix->par_d = 1;
706
  mix->setcaps = FALSE;
707
  mix->sendseg = FALSE;
708

709
  mix->segment_position = 0;
710
711
712
  gst_segment_init (&mix->segment, GST_FORMAT_TIME);

  gst_videomixer_reset_qos (mix);
713

714
715
  mix->fmt = GST_VIDEO_FORMAT_UNKNOWN;

716
  mix->last_ts = 0;
717
  mix->last_duration = -1;
718

719
720
721
  /* clean up collect data */
  walk = mix->collect->data;
  while (walk) {
722
    GstVideoMixerCollect *data = (GstVideoMixerCollect *) walk->data;
723
724
725
726

    gst_videomixer_collect_free (data);
    walk = g_slist_next (walk);
  }
727
728

  mix->next_sinkpad = 0;
729
  mix->flush_stop_pending = FALSE;
730
731
732
}

static void
733
gst_videomixer_init (GstVideoMixer * mix, GstVideoMixerClass * g_class)
734
735
736
737
738
739
{
  GstElementClass *klass = GST_ELEMENT_GET_CLASS (mix);

  mix->srcpad =
      gst_pad_new_from_template (gst_element_class_get_pad_template (klass,
          "src"), "src");
740
741
  gst_pad_set_getcaps_function (GST_PAD (mix->srcpad),
      GST_DEBUG_FUNCPTR (gst_videomixer_getcaps));
742
743
  gst_pad_set_setcaps_function (GST_PAD (mix->srcpad),
      GST_DEBUG_FUNCPTR (gst_videomixer_setcaps));
744
745
  gst_pad_set_query_function (GST_PAD (mix->srcpad),
      GST_DEBUG_FUNCPTR (gst_videomixer_query));
746
747
  gst_pad_set_event_function (GST_PAD (mix->srcpad),
      GST_DEBUG_FUNCPTR (gst_videomixer_src_event));
748
749
  gst_element_add_pad (GST_ELEMENT (mix), mix->srcpad);

750
  mix->collect = gst_collect_pads_new ();
751
752
  mix->background = DEFAULT_BACKGROUND;

753
  gst_collect_pads_set_function (mix->collect,
754
755
      (GstCollectPadsFunction) GST_DEBUG_FUNCPTR (gst_videomixer_collected),
      mix);
756

757
  mix->state_lock = g_mutex_new ();
758
759
760
761
762
763
764
765
766
767
  /* initialize variables */
  gst_videomixer_reset (mix);
}

static void
gst_videomixer_finalize (GObject * object)
{
  GstVideoMixer *mix = GST_VIDEO_MIXER (object);

  gst_object_unref (mix->collect);
768
  g_mutex_free (mix->state_lock);
769
770

  G_OBJECT_CLASS (parent_class)->finalize (object);
771
772
}

773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
static gboolean
gst_videomixer_query_duration (GstVideoMixer * mix, GstQuery * query)
{
  gint64 max;
  gboolean res;
  GstFormat format;
  GstIterator *it;
  gboolean done;

  /* parse format */
  gst_query_parse_duration (query, &format, NULL);

  max = -1;
  res = TRUE;
  done = FALSE;

  /* Take maximum of all durations */
  it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (mix));
  while (!done) {
    GstIteratorResult ires;
    gpointer item;

    ires = gst_iterator_next (it, &item);
    switch (ires) {
      case GST_ITERATOR_DONE:
        done = TRUE;
        break;
      case GST_ITERATOR_OK:
      {
        GstPad *pad = GST_PAD_CAST (item);
        gint64 duration;

        /* ask sink peer for duration */
        res &= gst_pad_query_peer_duration (pad, &format, &duration);
        /* take max from all valid return values */
        if (res) {
          /* valid unknown length, stop searching */
          if (duration == -1) {
            max = duration;
            done = TRUE;
          }
          /* else see if bigger than current max */
          else if (duration > max)
            max = duration;
        }
818
        gst_object_unref (pad);
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
        break;
      }
      case GST_ITERATOR_RESYNC:
        max = -1;
        res = TRUE;
        gst_iterator_resync (it);
        break;
      default:
        res = FALSE;
        done = TRUE;
        break;
    }
  }
  gst_iterator_free (it);

  if (res) {
    /* and store the max */
    GST_DEBUG_OBJECT (mix, "Total duration in format %s: %"
        GST_TIME_FORMAT, gst_format_get_name (format), GST_TIME_ARGS (max));
    gst_query_set_duration (query, format, max);
  }

  return res;
}

static gboolean
gst_videomixer_query_latency (GstVideoMixer * mix, GstQuery * query)
{
  GstClockTime min, max;
  gboolean live;
  gboolean res;
  GstIterator *it;
  gboolean done;

  res = TRUE;
  done = FALSE;
  live = FALSE;
  min = 0;
  max = GST_CLOCK_TIME_NONE;

  /* Take maximum of all latency values */
  it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (mix));
  while (!done) {
    GstIteratorResult ires;
    gpointer item;

    ires = gst_iterator_next (it, &item);
    switch (ires) {
      case GST_ITERATOR_DONE:
        done = TRUE;
        break;
      case GST_ITERATOR_OK:
      {
        GstPad *pad = GST_PAD_CAST (item);
873

874
        GstQuery *peerquery;
875

876
        GstClockTime min_cur, max_cur;
877

878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
        gboolean live_cur;

        peerquery = gst_query_new_latency ();

        /* Ask peer for latency */
        res &= gst_pad_peer_query (pad, peerquery);

        /* take max from all valid return values */
        if (res) {
          gst_query_parse_latency (peerquery, &live_cur, &min_cur, &max_cur);

          if (min_cur > min)
            min = min_cur;

          if (max_cur != GST_CLOCK_TIME_NONE &&
              ((max != GST_CLOCK_TIME_NONE && max_cur > max) ||
                  (max == GST_CLOCK_TIME_NONE)))
            max = max_cur;

          live = live || live_cur;
        }

        gst_query_unref (peerquery);
901
        gst_object_unref (pad);
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
        break;
      }
      case GST_ITERATOR_RESYNC:
        live = FALSE;
        min = 0;
        max = GST_CLOCK_TIME_NONE;
        res = TRUE;
        gst_iterator_resync (it);
        break;
      default:
        res = FALSE;
        done = TRUE;
        break;
    }
  }
  gst_iterator_free (it);

  if (res) {
    /* store the results */
    GST_DEBUG_OBJECT (mix, "Calculated total latency: live %s, min %"
        GST_TIME_FORMAT ", max %" GST_TIME_FORMAT,
        (live ? "yes" : "no"), GST_TIME_ARGS (min), GST_TIME_ARGS (max));
    gst_query_set_latency (query, live, min, max);
  }

  return res;
}

static gboolean
gst_videomixer_query (GstPad * pad, GstQuery * query)
{
  GstVideoMixer *mix = GST_VIDEO_MIXER (gst_pad_get_parent (pad));
  gboolean res = FALSE;

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_POSITION:
    {
      GstFormat format;

      gst_query_parse_position (query, &format, NULL);

      switch (format) {
        case GST_FORMAT_TIME:
          /* FIXME, bring to stream time, might be tricky */
          gst_query_set_position (query, format, mix->last_ts);
          res = TRUE;
          break;
        default:
          break;
      }
      break;
    }
    case GST_QUERY_DURATION:
      res = gst_videomixer_query_duration (mix, query);
      break;
    case GST_QUERY_LATENCY:
      res = gst_videomixer_query_latency (mix, query);
      break;
    default:
      /* FIXME, needs a custom query handler because we have multiple
962
963
       * sinkpads, send to the master pad until then */
      res = gst_pad_query (GST_PAD_CAST (mix->master), query);
964
965
966
967
968
969
970
      break;
  }

  gst_object_unref (mix);
  return res;
}

971
972
973
static GstCaps *
gst_videomixer_getcaps (GstPad * pad)
{
Sebastian Dröge's avatar
Sebastian Dröge committed
974
  GstVideoMixer *mix = GST_VIDEO_MIXER (gst_pad_get_parent (pad));
975
976
  GstCaps *caps;
  GstStructure *structure;
977
  int numCaps;
978

979
980
981
982
983
  if (mix->master) {
    caps =
        gst_caps_copy (gst_pad_get_pad_template_caps (GST_PAD (mix->master)));
  } else {
    caps = gst_caps_copy (gst_pad_get_pad_template_caps (mix->srcpad));
984
  }
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999

  numCaps = gst_caps_get_size (caps) - 1;
  for (; numCaps >= 0; numCaps--) {
    structure = gst_caps_get_structure (caps, numCaps);
    if (mix->out_width != 0) {
      gst_structure_set (structure, "width", G_TYPE_INT, mix->out_width, NULL);
    }
    if (mix->out_height != 0) {
      gst_structure_set (structure, "height", G_TYPE_INT, mix->out_height,
          NULL);
    }
    if (mix->fps_d != 0) {
      gst_structure_set (structure,
          "framerate", GST_TYPE_FRACTION, mix->fps_n, mix->fps_d, NULL);
    }
1000
1001
  }

1002
1003
  gst_object_unref (mix);

1004
1005
1006
  return caps;
}

1007
1008
1009
static gboolean
gst_videomixer_setcaps (GstPad * pad, GstCaps * caps)
{
1010
1011
1012
  GstVideoMixer *mixer = GST_VIDEO_MIXER (gst_pad_get_parent_element (pad));
  gboolean ret = FALSE;

1013
  GST_INFO_OBJECT (mixer, "set src caps: %" GST_PTR_FORMAT, caps);
1014

1015
1016
1017
1018
  mixer->blend = NULL;
  mixer->fill_checker = NULL;
  mixer->fill_color = NULL;

1019
  if (!gst_video_format_parse_caps (caps, &mixer->fmt, NULL, NULL))
1020
1021
    goto done;

1022
  switch (mixer->fmt) {
1023
1024
1025
1026
    case GST_VIDEO_FORMAT_AYUV:
      mixer->blend = gst_video_mixer_blend_ayuv;
      mixer->fill_checker = gst_video_mixer_fill_checker_ayuv;
      mixer->fill_color = gst_video_mixer_fill_color_ayuv;
1027
1028
      ret = TRUE;
      break;
1029
1030
1031
1032
    case GST_VIDEO_FORMAT_ARGB:
      mixer->blend = gst_video_mixer_blend_argb;
      mixer->fill_checker = gst_video_mixer_fill_checker_argb;
      mixer->fill_color = gst_video_mixer_fill_color_argb;
1033
1034
1035
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_BGRA:
1036
1037
1038
      mixer->blend = gst_video_mixer_blend_bgra;
      mixer->fill_checker = gst_video_mixer_fill_checker_bgra;
      mixer->fill_color = gst_video_mixer_fill_color_bgra;
1039
1040
      ret = TRUE;
      break;
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
    case GST_VIDEO_FORMAT_ABGR:
      mixer->blend = gst_video_mixer_blend_abgr;
      mixer->fill_checker = gst_video_mixer_fill_checker_abgr;
      mixer->fill_color = gst_video_mixer_fill_color_abgr;
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_RGBA:
      mixer->blend = gst_video_mixer_blend_rgba;
      mixer->fill_checker = gst_video_mixer_fill_checker_rgba;
      mixer->fill_color = gst_video_mixer_fill_color_rgba;
      ret = TRUE;
      break;
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
    case GST_VIDEO_FORMAT_Y444:
      mixer->blend = gst_video_mixer_blend_y444;
      mixer->fill_checker = gst_video_mixer_fill_checker_y444;
      mixer->fill_color = gst_video_mixer_fill_color_y444;
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_Y42B:
      mixer->blend = gst_video_mixer_blend_y42b;
      mixer->fill_checker = gst_video_mixer_fill_checker_y42b;
      mixer->fill_color = gst_video_mixer_fill_color_y42b;
      ret = TRUE;
      break;
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
    case GST_VIDEO_FORMAT_YUY2:
      mixer->blend = gst_video_mixer_blend_yuy2;
      mixer->fill_checker = gst_video_mixer_fill_checker_yuy2;
      mixer->fill_color = gst_video_mixer_fill_color_yuy2;
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_UYVY:
      mixer->blend = gst_video_mixer_blend_uyvy;
      mixer->fill_checker = gst_video_mixer_fill_checker_uyvy;
      mixer->fill_color = gst_video_mixer_fill_color_uyvy;
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_YVYU:
      mixer->blend = gst_video_mixer_blend_yvyu;
      mixer->fill_checker = gst_video_mixer_fill_checker_yvyu;
      mixer->fill_color = gst_video_mixer_fill_color_yvyu;
      ret = TRUE;
      break;
1083
1084
1085
1086
    case GST_VIDEO_FORMAT_I420:
      mixer->blend = gst_video_mixer_blend_i420;
      mixer->fill_checker = gst_video_mixer_fill_checker_i420;
      mixer->fill_color = gst_video_mixer_fill_color_i420;
1087
1088
      ret = TRUE;
      break;
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
    case GST_VIDEO_FORMAT_YV12:
      mixer->blend = gst_video_mixer_blend_yv12;
      mixer->fill_checker = gst_video_mixer_fill_checker_yv12;
      mixer->fill_color = gst_video_mixer_fill_color_yv12;
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_Y41B:
      mixer->blend = gst_video_mixer_blend_y41b;
      mixer->fill_checker = gst_video_mixer_fill_checker_y41b;
      mixer->fill_color = gst_video_mixer_fill_color_y41b;
      ret = TRUE;
      break;
1101
    case GST_VIDEO_FORMAT_RGB:
1102
1103
1104
      mixer->blend = gst_video_mixer_blend_rgb;
      mixer->fill_checker = gst_video_mixer_fill_checker_rgb;
      mixer->fill_color = gst_video_mixer_fill_color_rgb;
1105
1106
1107
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_BGR:
1108
1109
1110
      mixer->blend = gst_video_mixer_blend_bgr;
      mixer->fill_checker = gst_video_mixer_fill_checker_bgr;
      mixer->fill_color = gst_video_mixer_fill_color_bgr;
1111
1112
1113
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_xRGB:
1114
1115
1116
      mixer->blend = gst_video_mixer_blend_xrgb;
      mixer->fill_checker = gst_video_mixer_fill_checker_xrgb;
      mixer->fill_color = gst_video_mixer_fill_color_xrgb;
1117
1118
1119
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_xBGR:
1120
1121
1122
      mixer->blend = gst_video_mixer_blend_xbgr;
      mixer->fill_checker = gst_video_mixer_fill_checker_xbgr;
      mixer->fill_color = gst_video_mixer_fill_color_xbgr;
1123
1124
1125
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_RGBx:
1126
1127
1128
      mixer->blend = gst_video_mixer_blend_rgbx;
      mixer->fill_checker = gst_video_mixer_fill_checker_rgbx;
      mixer->fill_color = gst_video_mixer_fill_color_rgbx;
1129
1130
1131
      ret = TRUE;
      break;
    case GST_VIDEO_FORMAT_BGRx:
1132
1133
1134
      mixer->blend = gst_video_mixer_blend_bgrx;
      mixer->fill_checker = gst_video_mixer_fill_checker_bgrx;
      mixer->fill_color = gst_video_mixer_fill_color_bgrx;
1135
1136
      ret = TRUE;
      break;
1137
1138
    default:
      break;
1139
1140
  }

1141
1142
1143
1144
done:
  gst_object_unref (mixer);

  return ret;
1145
1146
}

1147
1148
1149
1150
static GstPad *
gst_videomixer_request_new_pad (GstElement * element,
    GstPadTemplate * templ, const gchar * req_name)
{
1151
1152
  GstVideoMixer *mix = NULL;
  GstVideoMixerPad *mixpad = NULL;
1153
1154
1155
1156
  GstElementClass *klass = GST_ELEMENT_GET_CLASS (element);

  g_return_val_if_fail (templ != NULL, NULL);

1157
1158
  if (G_UNLIKELY (templ->direction != GST_PAD_SINK)) {
    g_warning ("videomixer: request pad that is not a SINK pad");
1159
1160
1161
1162
1163
1164
1165
1166
    return NULL;
  }

  g_return_val_if_fail (GST_IS_VIDEO_MIXER (element), NULL);

  mix = GST_VIDEO_MIXER (element);

  if (templ == gst_element_class_get_pad_template (klass, "sink_%d")) {
1167
1168
1169
    gint serial = 0;
    gchar *name = NULL;
    GstVideoMixerCollect *mixcol = NULL;
1170

1171
1172
1173
    GST_VIDEO_MIXER_STATE_LOCK (mix);
    if (req_name == NULL || strlen (req_name) < 6
        || !g_str_has_prefix (req_name, "sink_")) {
1174
1175
      /* no name given when requesting the pad, use next available int */
      serial = mix->next_sinkpad++;
1176
1177
1178
    } else {
      /* parse serial number from requested padname */
      serial = atoi (&req_name[5]);
1179
1180
      if (serial >= mix->next_sinkpad)
        mix->next_sinkpad = serial + 1;
1181
    }
1182
    /* create new pad with the name */
1183
1184
1185
    name = g_strdup_printf ("sink_%d", serial);
    mixpad = g_object_new (GST_TYPE_VIDEO_MIXER_PAD, "name", name, "direction",
        templ->direction, "template", templ, NULL);
1186
1187
1188
    g_free (name);

    mixpad->zorder = mix->numpads;
1189
1190
1191
1192
1193
    mixpad->xpos = DEFAULT_PAD_XPOS;
    mixpad->ypos = DEFAULT_PAD_YPOS;
    mixpad->alpha = DEFAULT_PAD_ALPHA;

    mixcol = (GstVideoMixerCollect *)
1194
        gst_collect_pads_add_pad (mix->collect, GST_PAD (mixpad),
1195
1196
        sizeof (GstVideoMixerCollect));

1197
1198
1199
1200
1201
1202
1203
1204
    /* FIXME: hacked way to override/extend the event function of
     * GstCollectPads; because it sets its own event function giving the
     * element no access to events */
    mix->collect_event =
        (GstPadEventFunction) GST_PAD_EVENTFUNC (GST_PAD (mixpad));
    gst_pad_set_event_function (GST_PAD (mixpad),
        GST_DEBUG_FUNCPTR (gst_videomixer_sink_event));

1205
    /* Keep track of each other */
1206
1207
1208
1209
1210
    mixcol->mixpad = mixpad;
    mixpad->mixcol = mixcol;

    /* Keep an internal list of mixpads for zordering */
    mix->sinkpads = g_slist_append (mix->sinkpads, mixpad);
1211
    mix->numpads++;
1212
    GST_VIDEO_MIXER_STATE_UNLOCK (mix);
1213
  } else {
1214
    g_warning ("videomixer: this is not our template!");
1215
1216
1217
    return NULL;
  }

1218
  /* add the pad to the element */
1219
  gst_element_add_pad (element, GST_PAD (mixpad));
1220
  gst_child_proxy_child_added (GST_OBJECT (mix), GST_OBJECT (mixpad));
1221

1222
  return GST_PAD (mixpad);
1223
1224
}

1225
1226
1227
1228
static void
gst_videomixer_release_pad (GstElement * element, GstPad * pad)
{
  GstVideoMixer *mix = NULL;
1229
  GstVideoMixerPad *mixpad;
1230
1231

  mix = GST_VIDEO_MIXER (element);
1232
1233
1234
1235
1236
  GST_VIDEO_MIXER_STATE_LOCK (mix);
  if (G_UNLIKELY (g_slist_find (mix->sinkpads, pad) == NULL)) {
    g_warning ("Unknown pad %s", GST_PAD_NAME (pad));
    goto error;
  }
1237

1238
  mixpad = GST_VIDEO_MIXER_PAD (pad);
1239

1240
1241
1242
  mix->sinkpads = g_slist_remove (mix->sinkpads, pad);
  gst_videomixer_collect_free (mixpad->mixcol);
  gst_collect_pads_remove_pad (mix->collect, pad);
1243
  gst_child_proxy_child_removed (GST_OBJECT (mix), GST_OBJECT (mixpad));
1244
1245
1246
1247
  /* determine possibly new geometry and master */
  gst_videomixer_set_master_geometry (mix);
  mix->numpads--;
  GST_VIDEO_MIXER_STATE_UNLOCK (mix);
1248

1249
1250
1251
1252
  gst_element_remove_pad (element, pad);
  return;
error:
  GST_VIDEO_MIXER_STATE_UNLOCK (mix);
1253
1254
}

1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
static int
pad_zorder_compare (const GstVideoMixerPad * pad1,
    const GstVideoMixerPad * pad2)
{
  return pad1->zorder - pad2->zorder;
}

static void
gst_videomixer_sort_pads (GstVideoMixer * mix)
{
  mix->sinkpads = g_slist_sort (mix->sinkpads,
      (GCompareFunc) pad_zorder_compare);
}

1269
1270
1271
1272
1273
/* try to get a buffer on all pads. As long as the queued value is
 * negative, we skip buffers */
static gboolean
gst_videomixer_fill_queues (GstVideoMixer * mix)
{
1274
  GSList *walk = NULL;
1275
1276
  gboolean eos = TRUE;

1277
1278
1279
1280
  g_return_val_if_fail (GST_IS_VIDEO_MIXER (mix), FALSE);

  /* try to make sure we have a buffer from each usable pad first */
  walk = mix->collect->data;
1281
  while (walk) {
1282
1283
1284
    GstCollectData *data = (GstCollectData *) walk->data;
    GstVideoMixerCollect *mixcol = (GstVideoMixerCollect *) data;
    GstVideoMixerPad *mixpad = mixcol->mixpad;
1285
1286
1287

    walk = g_slist_next (walk);

1288
1289
    if (mixcol->buffer == NULL) {
      GstBuffer *buf = NULL;
1290

1291
      GST_LOG_OBJECT (mix,</