visual.c 17.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/* GStreamer
 * Copyright (C) 2004 Benjamin Otte <otte@gnome.org>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <gst/gst.h>
25
#include <gst/base/gstadapter.h>
26
27
28
29
30
31
32
33
34
35
36
37
38
39
#include <gst/video/video.h>
#include <gst/audio/audio.h>
#include <libvisual/libvisual.h>

#define GST_TYPE_VISUAL (gst_visual_get_type())
#define GST_IS_VISUAL(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VISUAL))
#define GST_VISUAL(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VISUAL,GstVisual))
#define GST_IS_VISUAL_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VISUAL))
#define GST_VISUAL_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VISUAL,GstVisualClass))
#define GST_VISUAL_GET_CLASS(obj)   (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_VISUAL, GstVisualClass))

typedef struct _GstVisual GstVisual;
typedef struct _GstVisualClass GstVisualClass;

40
41
42
GST_DEBUG_CATEGORY_STATIC (libvisual_debug);
#define GST_CAT_DEFAULT (libvisual_debug)

43
44
45
46
47
48
49
struct _GstVisual
{
  GstElement element;

  /* pads */
  GstPad *sinkpad;
  GstPad *srcpad;
50
  GstClockTime next_ts;
51
52
53
54
55
56
57

  /* libvisual stuff */
  VisAudio audio;
  VisVideo *video;
  VisActor *actor;

  /* audio/video state */
58
  gint rate;                    /* Input samplerate */
59
60
61
62

  /* framerate numerator & denominator */
  gint fps_n;
  gint fps_d;
63
64
  gint width;
  gint height;
65
66
67
68

  /* state stuff */
  GstAdapter *adapter;
  guint count;
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
};

struct _GstVisualClass
{
  GstElementClass parent_class;

  VisPluginRef *plugin;
};

GType gst_visual_get_type (void);


static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
84
    GST_STATIC_CAPS (GST_VIDEO_CAPS_xRGB_HOST_ENDIAN "; "
85
86
87
88
89
90
#if G_BYTE_ORDER == G_BIG_ENDIAN
        GST_VIDEO_CAPS_RGB "; "
#else
        GST_VIDEO_CAPS_BGR "; "
#endif
        GST_VIDEO_CAPS_RGB_16)
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    );

static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
    GST_PAD_SINK,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS ("audio/x-raw-int, "
        "width = (int) 16, "
        "depth = (int) 16, "
        "endianness = (int) BYTE_ORDER, "
        "signed = (boolean) TRUE, "
        "channels = (int) 2, " "rate = (int) [ 1000, MAX ]")
    );


static void gst_visual_class_init (gpointer g_class, gpointer class_data);
static void gst_visual_init (GstVisual * visual);
107
static void gst_visual_dispose (GObject * object);
108

109
110
static GstStateChangeReturn gst_visual_change_state (GstElement * element,
    GstStateChange transition);
111
static GstFlowReturn gst_visual_chain (GstPad * pad, GstBuffer * buffer);
112

113
114
static gboolean gst_visual_sink_setcaps (GstPad * pad, GstCaps * caps);
static gboolean gst_visual_src_setcaps (GstPad * pad, GstCaps * caps);
115
static GstCaps *gst_visual_getcaps (GstPad * pad);
116
117
static void libvisual_log_handler (const char *message, const char *funcname,
    void *priv);
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

static GstElementClass *parent_class = NULL;

GType
gst_visual_get_type (void)
{
  static GType type = 0;

  if (!type) {
    static const GTypeInfo info = {
      sizeof (GstVisualClass),
      NULL,
      NULL,
      gst_visual_class_init,
      NULL,
      NULL,
      sizeof (GstVisual),
      0,
      (GInstanceInitFunc) gst_visual_init,
    };

    type = g_type_register_static (GST_TYPE_ELEMENT, "GstVisual", &info, 0);
  }
  return type;
}

144
145
146
147
148
149
150
static void
libvisual_log_handler (const char *message, const char *funcname, void *priv)
{
  GST_CAT_LEVEL_LOG (libvisual_debug, (GstDebugLevel) (priv), NULL, "%s - %s",
      funcname, message);
}

151
152
153
154
static void
gst_visual_class_init (gpointer g_class, gpointer class_data)
{
  GstVisualClass *klass = GST_VISUAL_CLASS (g_class);
155
156
  GstElementClass *element = GST_ELEMENT_CLASS (g_class);
  GObjectClass *object = G_OBJECT_CLASS (g_class);
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183

  klass->plugin = class_data;

  element->change_state = gst_visual_change_state;

  if (class_data == NULL) {
    parent_class = g_type_class_peek_parent (g_class);
  } else {
    GstElementDetails details = {
      NULL,
      "Visualization",
      klass->plugin->info->about,
      "Benjamin Otte <otte@gnome.org>"
    };

    details.longname = g_strdup_printf ("libvisual %s plugin v.%s",
        klass->plugin->info->name, klass->plugin->info->version);

    /* FIXME: improve to only register what plugin supports? */
    gst_element_class_add_pad_template (element,
        gst_static_pad_template_get (&src_template));
    gst_element_class_add_pad_template (element,
        gst_static_pad_template_get (&sink_template));
    gst_element_class_set_details (element, &details);
    g_free (details.longname);
  }

184
  object->dispose = gst_visual_dispose;
185
186
187
188
189
190
}

static void
gst_visual_init (GstVisual * visual)
{
  /* create the sink and src pads */
191
  visual->sinkpad = gst_pad_new_from_static_template (&sink_template, "sink");
192
  gst_pad_set_setcaps_function (visual->sinkpad, gst_visual_sink_setcaps);
193
194
195
  gst_pad_set_chain_function (visual->sinkpad, gst_visual_chain);
  gst_element_add_pad (GST_ELEMENT (visual), visual->sinkpad);

196
  visual->srcpad = gst_pad_new_from_static_template (&src_template, "src");
197
  gst_pad_set_setcaps_function (visual->srcpad, gst_visual_src_setcaps);
198
199
200
  gst_pad_set_getcaps_function (visual->srcpad, gst_visual_getcaps);
  gst_element_add_pad (GST_ELEMENT (visual), visual->srcpad);

201
  visual->next_ts = 0;
202
203
204
205
206
207
208
209
210
211
212
213
214
  visual->adapter = gst_adapter_new ();
}

static void
gst_visual_dispose (GObject * object)
{
  GstVisual *visual = GST_VISUAL (object);

  if (visual->adapter) {
    g_object_unref (visual->adapter);
    visual->adapter = NULL;
  }

215
216
217
218
219
220
221
222
223
  if (visual->actor) {
    visual_object_unref (VISUAL_OBJECT (visual->actor));
    visual->actor = NULL;
  }

  if (visual->video) {
    visual_object_unref (VISUAL_OBJECT (visual->video));
    visual->video = NULL;
  }
224
  GST_CALL_PARENT (G_OBJECT_CLASS, dispose, (object));
225
226
227
228
229
230
231
}

static GstCaps *
gst_visual_getcaps (GstPad * pad)
{
  GstCaps *ret;
  GstVisual *visual = GST_VISUAL (gst_pad_get_parent (pad));
232
  int depths;
233

234
235
236
237
  if (!visual->actor) {
    ret = gst_caps_copy (gst_pad_get_pad_template_caps (visual->srcpad));
    goto beach;
  }
238
239

  ret = gst_caps_new_empty ();
240
241
242
  depths = visual_actor_get_supported_depth (visual->actor);
  if (depths < 0) {
    /* FIXME: set an error */
243
    goto beach;
244
245
246
  }
  if (depths == VISUAL_VIDEO_DEPTH_GL) {
    /* We can't handle GL only plugins */
247
    goto beach;
248
  }
249

250
251
252
  GST_DEBUG_OBJECT (visual, "libvisual plugin supports depths %u (0x%04x)",
      depths, depths);
  /* if (depths & VISUAL_VIDEO_DEPTH_32BIT) Always supports 32bit output */
253
254
255
  gst_caps_append (ret, gst_caps_from_string (GST_VIDEO_CAPS_xRGB_HOST_ENDIAN));

  if (depths & VISUAL_VIDEO_DEPTH_24BIT) {
256
257
258
#if G_BYTE_ORDER == G_BIG_ENDIAN
    gst_caps_append (ret, gst_caps_from_string (GST_VIDEO_CAPS_RGB));
#else
259
    gst_caps_append (ret, gst_caps_from_string (GST_VIDEO_CAPS_BGR));
260
#endif
261
  }
262
  if (depths & VISUAL_VIDEO_DEPTH_16BIT) {
263
264
    gst_caps_append (ret, gst_caps_from_string (GST_VIDEO_CAPS_RGB_16));
  }
265

266
267
beach:

268
  GST_DEBUG_OBJECT (visual, "returning caps %" GST_PTR_FORMAT, ret);
269
  gst_object_unref (visual);
270
271
272
  return ret;
}

273
static gboolean
274
gst_visual_src_setcaps (GstPad * pad, GstCaps * caps)
275
276
277
278
279
280
281
{
  GstVisual *visual = GST_VISUAL (gst_pad_get_parent (pad));
  GstStructure *structure;
  gint depth;

  structure = gst_caps_get_structure (caps, 0);

282
283
  GST_DEBUG_OBJECT (visual, "src pad got caps %" GST_PTR_FORMAT, caps);

284
  if (!gst_structure_get_int (structure, "width", &visual->width))
285
    goto error;
286
  if (!gst_structure_get_int (structure, "height", &visual->height))
287
    goto error;
288
  if (!gst_structure_get_int (structure, "bpp", &depth))
289
    goto error;
Jan Schmidt's avatar
Jan Schmidt committed
290
291
  if (!gst_structure_get_fraction (structure, "framerate", &visual->fps_n,
          &visual->fps_d))
292
    goto error;
293

294
  visual_video_set_depth (visual->video,
295
296
      visual_video_depth_enum_from_value (depth));
  visual_video_set_dimension (visual->video, visual->width, visual->height);
297
  visual_actor_video_negotiate (visual->actor, 0, FALSE, FALSE);
298

299
  gst_object_unref (visual);
300
  return TRUE;
301
302
303
304
305
306

error:
  {
    gst_object_unref (visual);
    return FALSE;
  }
307
308
}

309
static gboolean
310
gst_visual_sink_setcaps (GstPad * pad, GstCaps * caps)
311
312
313
314
315
316
317
318
{
  GstVisual *visual = GST_VISUAL (gst_pad_get_parent (pad));
  GstStructure *structure;

  structure = gst_caps_get_structure (caps, 0);

  gst_structure_get_int (structure, "rate", &visual->rate);

319
  gst_object_unref (visual);
320
321
322
  return TRUE;
}

323
324
static gboolean
gst_vis_src_negotiate (GstVisual * visual)
325
{
326
327
328
  GstCaps *othercaps, *target, *intersect;
  GstStructure *structure;
  const GstCaps *templ;
329

330
  templ = gst_pad_get_pad_template_caps (visual->srcpad);
331

332
333
334
335
336
  /* see what the peer can do */
  othercaps = gst_pad_peer_get_caps (visual->srcpad);
  if (othercaps) {
    intersect = gst_caps_intersect (othercaps, templ);
    gst_caps_unref (othercaps);
337

338
339
    if (gst_caps_is_empty (intersect))
      goto no_format;
340

341
342
343
344
345
    target = gst_caps_copy_nth (intersect, 0);
    gst_caps_unref (intersect);
  } else {
    target = gst_caps_ref ((GstCaps *) templ);
  }
346

347
348
349
350
  structure = gst_caps_get_structure (target, 0);
  gst_structure_fixate_field_nearest_int (structure, "width", 320);
  gst_structure_fixate_field_nearest_int (structure, "height", 240);
  gst_structure_fixate_field_nearest_fraction (structure, "framerate", 30, 1);
351

352
353
  gst_pad_set_caps (visual->srcpad, target);
  gst_caps_unref (target);
354

355
  return TRUE;
356

357
358
359
360
361
362
363
no_format:
  {
    gst_caps_unref (intersect);
    gst_caps_unref (othercaps);
    return FALSE;
  }
}
364

365
366
367
368
static GstFlowReturn
get_buffer (GstVisual * visual, GstBuffer ** outbuf)
{
  GstFlowReturn ret;
369

370
371
372
373
  if (GST_PAD_CAPS (visual->srcpad) == NULL) {
    if (!gst_vis_src_negotiate (visual))
      return GST_FLOW_NOT_NEGOTIATED;
  }
374

375
376
  GST_DEBUG_OBJECT (visual, "allocating output buffer with caps %"
      GST_PTR_FORMAT, GST_PAD_CAPS (visual->srcpad));
377

378
379
380
381
382
  ret =
      gst_pad_alloc_buffer_and_set_caps (visual->srcpad,
      GST_BUFFER_OFFSET_NONE,
      visual->video->height * GST_ROUND_UP_4 (visual->video->width) *
      visual->video->bpp, GST_PAD_CAPS (visual->srcpad), outbuf);
383

384
385
386
  if (ret != GST_FLOW_OK)
    return ret;

387
388
389
  if (*outbuf == NULL)
    return GST_FLOW_ERROR;

390
  return GST_FLOW_OK;
391
392
}

393
394
static GstFlowReturn
gst_visual_chain (GstPad * pad, GstBuffer * buffer)
395
{
396
  GstBuffer *outbuf = NULL;
397
  guint i;
398
  GstVisual *visual = GST_VISUAL (gst_pad_get_parent (pad));
399
400
401
  GstFlowReturn ret = GST_FLOW_OK;
  guint spf;

402
403
  GST_DEBUG_OBJECT (visual, "chain function called");

404
405
406
407
408
409
  /* If we don't have an output format yet, preallocate a buffer to try and
   * set one */
  if (GST_PAD_CAPS (visual->srcpad) == NULL) {
    ret = get_buffer (visual, &outbuf);
    if (ret != GST_FLOW_OK) {
      gst_buffer_unref (buffer);
410
      goto beach;
411
412
    }
  }
413

414
  /* Match timestamps from the incoming audio */
415
416
417
  if (GST_BUFFER_TIMESTAMP (buffer) != GST_CLOCK_TIME_NONE)
    visual->next_ts = GST_BUFFER_TIMESTAMP (buffer);

418
  /* spf = samples per frame */
419
  spf = ((guint64) (visual->rate) * visual->fps_d) / visual->fps_n;
420
  gst_adapter_push (visual->adapter, buffer);
421
422
423
424

  while (gst_adapter_available (visual->adapter) > MAX (512, spf) * 4 &&
      (ret == GST_FLOW_OK)) {
    /* Read 512 samples per channel */
425
    const guint16 *data =
426
        (const guint16 *) gst_adapter_peek (visual->adapter, 512 * 4);
427

428
429
430
431
    for (i = 0; i < 512; i++) {
      visual->audio.plugpcm[0][i] = *data++;
      visual->audio.plugpcm[1][i] = *data++;
    }
432
433
434
435

    if (outbuf == NULL) {
      ret = get_buffer (visual, &outbuf);
      if (ret != GST_FLOW_OK) {
436
        goto beach;
437
438
439
440
441
442
443
444
      }
    }

    if (visual->video != NULL) {
      visual_video_set_buffer (visual->video, GST_BUFFER_DATA (outbuf));
      visual_audio_analyze (&visual->audio);
      visual_actor_run (visual->actor, &visual->audio);

445
      GST_BUFFER_TIMESTAMP (outbuf) = visual->next_ts;
Jan Schmidt's avatar
Jan Schmidt committed
446
      GST_BUFFER_DURATION (outbuf) = gst_util_uint64_scale_int (GST_SECOND,
447
          visual->fps_d, visual->fps_n);
448
      visual->next_ts += GST_BUFFER_DURATION (outbuf);
449
450
451
452
453
454
      ret = gst_pad_push (visual->srcpad, outbuf);
      outbuf = NULL;
    }

    /* Flush out the number of samples per frame * channels * sizeof (gint16) */
    /* Recompute spf in case caps changed */
455
    spf = ((guint64) (visual->rate) * visual->fps_d) / visual->fps_n;
456
457
458
459
    GST_DEBUG_OBJECT (visual, "finished frame, flushing %u samples from input",
        spf);
    gst_adapter_flush (visual->adapter,
        MIN (gst_adapter_available (visual->adapter), spf * 4));
460
  }
461

462
  /* so we're on the safe side */
463
464
  if (visual->video)
    visual_video_set_buffer (visual->video, NULL);
465

466
467
468
  if (outbuf != NULL)
    gst_buffer_unref (outbuf);

469
470
471
beach:
  gst_object_unref (visual);

472
  return ret;
473
474
}

475
476
static GstStateChangeReturn
gst_visual_change_state (GstElement * element, GstStateChange transition)
477
478
{
  GstVisual *visual = GST_VISUAL (element);
479
  GstStateChangeReturn ret;
480

481
482
  switch (transition) {
    case GST_STATE_CHANGE_NULL_TO_READY:
483
      visual->actor =
484
485
          visual_actor_new (GST_VISUAL_GET_CLASS (visual)->plugin->info->
          plugname);
486
487
488
      visual->video = visual_video_new ();

      if (!visual->actor || !visual->video)
489
        return GST_STATE_CHANGE_FAILURE;
490

491
      if (visual_actor_realize (visual->actor) != 0) {
492
        visual_object_unref (VISUAL_OBJECT (visual->actor));
493
        visual->actor = NULL;
494
        return GST_STATE_CHANGE_FAILURE;
495
      }
496
      visual_actor_set_video (visual->actor, visual->video);
497
      break;
498
    case GST_STATE_CHANGE_READY_TO_PAUSED:
499
      gst_adapter_clear (visual->adapter);
500
      break;
501
    case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
502
      break;
503
504
505
506
507
508
509
    default:
      break;
  }

  ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);

  switch (transition) {
510
    case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
511
      break;
512
    case GST_STATE_CHANGE_PAUSED_TO_READY:
513
      visual->next_ts = 0;
514
      break;
515
    case GST_STATE_CHANGE_READY_TO_NULL:
516
517
518
519
      if (visual->actor)
        visual_object_unref (VISUAL_OBJECT (visual->actor));
      if (visual->video)
        visual_object_unref (VISUAL_OBJECT (visual->video));
520
      visual->actor = NULL;
521
      visual->video = NULL;
522
523
      break;
    default:
524
      break;
525
526
  }

527
  return ret;
528
529
}

530
531
532
533
534
535
static void
make_valid_name (char *name)
{
  /*
   * Replace invalid chars with _ in the type name
   */
536
  static const gchar extra_chars[] = "-_+";
537
538
539
540
541
542
543
544
545
546
547
  gchar *p = name;

  for (; *p; p++) {
    int valid = ((p[0] >= 'A' && p[0] <= 'Z') ||
        (p[0] >= 'a' && p[0] <= 'z') ||
        (p[0] >= '0' && p[0] <= '9') || strchr (extra_chars, p[0]));
    if (!valid)
      *p = '_';
  }
}

548
549
550
551
552
553
static gboolean
plugin_init (GstPlugin * plugin)
{
  guint i;
  VisList *list;

554
555
556
557
558
559
560
561
562
563
564
565
  GST_DEBUG_CATEGORY_INIT (libvisual_debug, "libvisual", 0,
      "libvisual audio visualisations");

  visual_log_set_verboseness (VISUAL_LOG_VERBOSENESS_LOW);
  visual_log_set_info_handler (libvisual_log_handler, (void *) GST_LEVEL_INFO);
  visual_log_set_warning_handler (libvisual_log_handler,
      (void *) GST_LEVEL_WARNING);
  visual_log_set_critical_handler (libvisual_log_handler,
      (void *) GST_LEVEL_ERROR);
  visual_log_set_error_handler (libvisual_log_handler,
      (void *) GST_LEVEL_ERROR);

566
567
568
569
  if (!visual_is_initialized ())
    if (visual_init (NULL, NULL) != 0)
      return FALSE;

570
571
572
  list = visual_actor_get_list ();
  for (i = 0; i < visual_list_count (list); i++) {
    VisPluginRef *ref = visual_list_get (list, i);
573
574
575
    VisActorPlugin *actplugin = NULL;
    VisPluginData *visplugin = NULL;
    gboolean is_gl = FALSE;
576
577
578
579
580
581
582
583
584
585
586
587
588
589
    GType type;
    gchar *name;
    GTypeInfo info = {
      sizeof (GstVisualClass),
      NULL,
      NULL,
      gst_visual_class_init,
      NULL,
      ref,
      sizeof (GstVisual),
      0,
      NULL
    };

590
591
    visplugin = visual_plugin_load (ref);

592
    if (ref->info->plugname == NULL)
593
      continue;
594

595
596
    actplugin = VISUAL_PLUGIN_ACTOR (visplugin->info->plugin);

597
598
    /* Ignore plugins that only support GL output for now */
    if (actplugin->depth == VISUAL_VIDEO_DEPTH_GL) {
599
600
601
602
603
604
605
      GST_DEBUG ("plugin %s is a GL plugin (%d), ignoring",
          ref->info->plugname, actplugin->depth);
      is_gl = TRUE;
    } else {
      GST_DEBUG ("plugin %s is not a GL plugin (%d), registering",
          ref->info->plugname, actplugin->depth);
    }
606

607
608
609
610
611
612
613
614
615
616
617
618
619
620
    visual_plugin_unload (visplugin);

    if (!is_gl) {
      name = g_strdup_printf ("GstVisual%s", ref->info->plugname);
      make_valid_name (name);
      type = g_type_register_static (GST_TYPE_VISUAL, name, &info, 0);
      g_free (name);

      name = g_strdup_printf ("libvisual_%s", ref->info->plugname);
      make_valid_name (name);
      if (!gst_element_register (plugin, name, GST_RANK_NONE, type)) {
        g_free (name);
        return FALSE;
      }
621
622
623
624
625
626
627
628
629
630
631
      g_free (name);
    }
  }

  return TRUE;
}

GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
    GST_VERSION_MINOR,
    "libvisual",
    "libvisual visualization plugins",
632
    plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)