gstgloverlay.c 25.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * GStreamer
 * Copyright (C) 2008 Filippo Argiolas <filippo.argiolas@gmail.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/**
 * SECTION:element-gloverlay
23
 * @title: gloverlay
24 25 26
 *
 * Overlay GL video texture with a PNG image
 *
27
 * ## Examples
28
 * |[
29
 * gst-launch-1.0 videotestsrc ! gloverlay location=image.jpg ! glimagesink
30 31
 * ]|
 * FBO (Frame Buffer Object) is required.
32
 *
33 34 35 36 37 38
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

39
#include <gst/base/gsttypefindhelper.h>
40 41 42 43
#include <gst/gl/gstglconfig.h>

#include "gstgloverlay.h"
#include "effects/gstgleffectssources.h"
44
#include "gstglutils.h"
45 46 47

#include <stdio.h>
#include <stdlib.h>
48 49 50
#ifdef _MSC_VER
#define HAVE_BOOLEAN
#endif
51 52 53 54 55 56 57 58 59 60 61 62 63 64
#include <jpeglib.h>
#include <png.h>

#if PNG_LIBPNG_VER >= 10400
#define int_p_NULL         NULL
#define png_infopp_NULL    NULL
#endif

#define GST_CAT_DEFAULT gst_gl_overlay_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);

#define DEBUG_INIT \
  GST_DEBUG_CATEGORY_INIT (gst_gl_overlay_debug, "gloverlay", 0, "gloverlay element");

65
#define gst_gl_overlay_parent_class parent_class
66 67 68 69 70 71 72 73 74 75 76
G_DEFINE_TYPE_WITH_CODE (GstGLOverlay, gst_gl_overlay, GST_TYPE_GL_FILTER,
    DEBUG_INIT);

static gboolean gst_gl_overlay_set_caps (GstGLFilter * filter,
    GstCaps * incaps, GstCaps * outcaps);

static void gst_gl_overlay_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_gl_overlay_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);

77 78
static void gst_gl_overlay_before_transform (GstBaseTransform * trans,
    GstBuffer * outbuf);
79
static gboolean gst_gl_overlay_filter_texture (GstGLFilter * filter,
80
    GstGLMemory * in_tex, GstGLMemory * out_tex);
81

82 83
static gboolean gst_gl_overlay_load_png (GstGLOverlay * overlay, FILE * fp);
static gboolean gst_gl_overlay_load_jpeg (GstGLOverlay * overlay, FILE * fp);
84 85 86 87 88

enum
{
  PROP_0,
  PROP_LOCATION,
89 90 91 92 93 94 95
  PROP_OFFSET_X,
  PROP_OFFSET_Y,
  PROP_RELATIVE_X,
  PROP_RELATIVE_Y,
  PROP_OVERLAY_WIDTH,
  PROP_OVERLAY_HEIGHT,
  PROP_ALPHA
96 97
};

98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
/* *INDENT-OFF* */
/* vertex source */
static const gchar *overlay_v_src =
    "attribute vec4 a_position;\n"
    "attribute vec2 a_texcoord;\n"
    "varying vec2 v_texcoord;\n"
    "void main()\n"
    "{\n"
    "   gl_Position = a_position;\n"
    "   v_texcoord = a_texcoord;\n"
    "}";

/* fragment source */
static const gchar *overlay_f_src =
    "#ifdef GL_ES\n"
    "precision mediump float;\n"
    "#endif\n"
    "uniform sampler2D texture;\n"
    "uniform float alpha;\n"
    "varying vec2 v_texcoord;\n"
    "void main()\n"
    "{\n"
    "  vec4 rgba = texture2D( texture, v_texcoord );\n"
    "  gl_FragColor = vec4(rgba.rgb, rgba.a * alpha);\n"
    "}\n";
/* *INDENT-ON* */
124 125

/* init resources that need a gl context */
126
static gboolean
127
gst_gl_overlay_gl_start (GstGLBaseFilter * base_filter)
128
{
129
  GstGLOverlay *overlay = GST_GL_OVERLAY (base_filter);
130

131 132
  if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base_filter))
    return FALSE;
133

134 135
  return gst_gl_context_gen_shader (base_filter->context, overlay_v_src,
      overlay_f_src, &overlay->shader);
136 137 138 139
}

/* free resources that need a gl context */
static void
140
gst_gl_overlay_gl_stop (GstGLBaseFilter * base_filter)
141
{
142 143
  GstGLOverlay *overlay = GST_GL_OVERLAY (base_filter);
  const GstGLFuncs *gl = base_filter->context->gl_vtable;
144

145
  if (overlay->shader) {
146
    gst_object_unref (overlay->shader);
147 148 149 150 151 152 153
    overlay->shader = NULL;
  }

  if (overlay->image_memory) {
    gst_memory_unref ((GstMemory *) overlay->image_memory);
    overlay->image_memory = NULL;
  }
Matthew Waters's avatar
Matthew Waters committed
154 155 156 157 158 159 160 161 162 163 164

  if (overlay->vao) {
    gl->DeleteVertexArrays (1, &overlay->vao);
    overlay->vao = 0;
  }

  if (overlay->vbo) {
    gl->DeleteBuffers (1, &overlay->vbo);
    overlay->vbo = 0;
  }

165 166 167 168 169
  if (overlay->vbo_indices) {
    gl->DeleteBuffers (1, &overlay->vbo_indices);
    overlay->vbo_indices = 0;
  }

Matthew Waters's avatar
Matthew Waters committed
170 171 172 173 174 175 176 177 178
  if (overlay->overlay_vao) {
    gl->DeleteVertexArrays (1, &overlay->overlay_vao);
    overlay->overlay_vao = 0;
  }

  if (overlay->overlay_vbo) {
    gl->DeleteBuffers (1, &overlay->overlay_vbo);
    overlay->overlay_vbo = 0;
  }
179 180

  GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base_filter);
181 182 183 184 185 186 187 188 189 190 191
}

static void
gst_gl_overlay_class_init (GstGLOverlayClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *element_class;

  gobject_class = (GObjectClass *) klass;
  element_class = GST_ELEMENT_CLASS (klass);

192 193
  gst_gl_filter_add_rgba_pad_templates (GST_GL_FILTER_CLASS (klass));

194 195 196
  gobject_class->set_property = gst_gl_overlay_set_property;
  gobject_class->get_property = gst_gl_overlay_get_property;

197 198 199
  GST_GL_BASE_FILTER_CLASS (klass)->gl_start = gst_gl_overlay_gl_start;
  GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = gst_gl_overlay_gl_stop;

200 201
  GST_GL_FILTER_CLASS (klass)->set_caps = gst_gl_overlay_set_caps;
  GST_GL_FILTER_CLASS (klass)->filter_texture = gst_gl_overlay_filter_texture;
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252

  GST_BASE_TRANSFORM_CLASS (klass)->before_transform =
      GST_DEBUG_FUNCPTR (gst_gl_overlay_before_transform);

  g_object_class_install_property (gobject_class, PROP_LOCATION,
      g_param_spec_string ("location", "location",
          "Location of image file to overlay", NULL, GST_PARAM_CONTROLLABLE
          | GST_PARAM_MUTABLE_PLAYING | G_PARAM_READWRITE
          | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_OFFSET_X,
      g_param_spec_int ("offset-x", "X Offset",
          "For positive value, horizontal offset of overlay image in pixels from"
          " left of video image. For negative value, horizontal offset of overlay"
          " image in pixels from right of video image", G_MININT, G_MAXINT, 0,
          GST_PARAM_CONTROLLABLE | GST_PARAM_MUTABLE_PLAYING | G_PARAM_READWRITE
          | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_OFFSET_Y,
      g_param_spec_int ("offset-y", "Y Offset",
          "For positive value, vertical offset of overlay image in pixels from"
          " top of video image. For negative value, vertical offset of overlay"
          " image in pixels from bottom of video image", G_MININT, G_MAXINT, 0,
          GST_PARAM_CONTROLLABLE | GST_PARAM_MUTABLE_PLAYING | G_PARAM_READWRITE
          | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_RELATIVE_X,
      g_param_spec_double ("relative-x", "Relative X Offset",
          "Horizontal offset of overlay image in fractions of video image "
          "width, from top-left corner of video image", 0.0, 1.0, 0.0,
          GST_PARAM_CONTROLLABLE | GST_PARAM_MUTABLE_PLAYING | G_PARAM_READWRITE
          | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_RELATIVE_Y,
      g_param_spec_double ("relative-y", "Relative Y Offset",
          "Vertical offset of overlay image in fractions of video image "
          "height, from top-left corner of video image", 0.0, 1.0, 0.0,
          GST_PARAM_CONTROLLABLE | GST_PARAM_MUTABLE_PLAYING | G_PARAM_READWRITE
          | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_OVERLAY_WIDTH,
      g_param_spec_int ("overlay-width", "Overlay Width",
          "Width of overlay image in pixels (0 = same as overlay image)", 0,
          G_MAXINT, 0,
          GST_PARAM_CONTROLLABLE | GST_PARAM_MUTABLE_PLAYING | G_PARAM_READWRITE
          | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_OVERLAY_HEIGHT,
      g_param_spec_int ("overlay-height", "Overlay Height",
          "Height of overlay image in pixels (0 = same as overlay image)", 0,
          G_MAXINT, 0,
          GST_PARAM_CONTROLLABLE | GST_PARAM_MUTABLE_PLAYING | G_PARAM_READWRITE
          | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_ALPHA,
      g_param_spec_double ("alpha", "Alpha", "Global alpha of overlay image",
          0.0, 1.0, 1.0, GST_PARAM_CONTROLLABLE | GST_PARAM_MUTABLE_PLAYING
          | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
253 254 255

  gst_element_class_set_metadata (element_class,
      "Gstreamer OpenGL Overlay", "Filter/Effect/Video",
256 257 258
      "Overlay GL video texture with a JPEG/PNG image",
      "Filippo Argiolas <filippo.argiolas@gmail.com>, "
      "Matthew Waters <matthew@centricular.com>");
259

260
  GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
261
      GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3;
262 263 264 265 266
}

static void
gst_gl_overlay_init (GstGLOverlay * overlay)
{
267 268 269 270 271 272 273 274 275 276
  overlay->offset_x = 0;
  overlay->offset_y = 0;

  overlay->relative_x = 0.0;
  overlay->relative_y = 0.0;

  overlay->overlay_width = 0;
  overlay->overlay_height = 0;

  overlay->alpha = 1.0;
277 278 279 280 281 282 283 284 285 286
}

static void
gst_gl_overlay_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstGLOverlay *overlay = GST_GL_OVERLAY (object);

  switch (prop_id) {
    case PROP_LOCATION:
287
      g_free (overlay->location);
288
      overlay->location_has_changed = TRUE;
289 290
      overlay->location = g_value_dup_string (value);
      break;
291 292
    case PROP_OFFSET_X:
      overlay->offset_x = g_value_get_int (value);
Matthew Waters's avatar
Matthew Waters committed
293
      overlay->geometry_change = TRUE;
294
      break;
295 296
    case PROP_OFFSET_Y:
      overlay->offset_y = g_value_get_int (value);
Matthew Waters's avatar
Matthew Waters committed
297
      overlay->geometry_change = TRUE;
298
      break;
299 300
    case PROP_RELATIVE_X:
      overlay->relative_x = g_value_get_double (value);
Matthew Waters's avatar
Matthew Waters committed
301
      overlay->geometry_change = TRUE;
302
      break;
303 304
    case PROP_RELATIVE_Y:
      overlay->relative_y = g_value_get_double (value);
Matthew Waters's avatar
Matthew Waters committed
305
      overlay->geometry_change = TRUE;
306
      break;
307 308
    case PROP_OVERLAY_WIDTH:
      overlay->overlay_width = g_value_get_int (value);
Matthew Waters's avatar
Matthew Waters committed
309
      overlay->geometry_change = TRUE;
310
      break;
311 312
    case PROP_OVERLAY_HEIGHT:
      overlay->overlay_height = g_value_get_int (value);
Matthew Waters's avatar
Matthew Waters committed
313
      overlay->geometry_change = TRUE;
314
      break;
315 316
    case PROP_ALPHA:
      overlay->alpha = g_value_get_double (value);
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_gl_overlay_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
  GstGLOverlay *overlay = GST_GL_OVERLAY (object);

  switch (prop_id) {
    case PROP_LOCATION:
      g_value_set_string (value, overlay->location);
      break;
334 335
    case PROP_OFFSET_X:
      g_value_set_int (value, overlay->offset_x);
336
      break;
337 338
    case PROP_OFFSET_Y:
      g_value_set_int (value, overlay->offset_y);
339
      break;
340 341
    case PROP_RELATIVE_X:
      g_value_set_double (value, overlay->relative_x);
342
      break;
343 344
    case PROP_RELATIVE_Y:
      g_value_set_double (value, overlay->relative_y);
345
      break;
346 347
    case PROP_OVERLAY_WIDTH:
      g_value_set_int (value, overlay->overlay_width);
348
      break;
349 350
    case PROP_OVERLAY_HEIGHT:
      g_value_set_int (value, overlay->overlay_height);
351
      break;
352 353
    case PROP_ALPHA:
      g_value_set_double (value, overlay->alpha);
354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static gboolean
gst_gl_overlay_set_caps (GstGLFilter * filter, GstCaps * incaps,
    GstCaps * outcaps)
{
  GstGLOverlay *overlay = GST_GL_OVERLAY (filter);
  GstStructure *s = gst_caps_get_structure (incaps, 0);
  gint width = 0;
  gint height = 0;

  gst_structure_get_int (s, "width", &width);
  gst_structure_get_int (s, "height", &height);

373 374
  overlay->window_width = width;
  overlay->window_height = height;
375 376 377 378

  return TRUE;
}

Matthew Waters's avatar
Matthew Waters committed
379 380 381
static void
_unbind_buffer (GstGLOverlay * overlay)
{
382
  GstGLFilter *filter = GST_GL_FILTER (overlay);
383
  const GstGLFuncs *gl = GST_GL_BASE_FILTER (overlay)->context->gl_vtable;
Matthew Waters's avatar
Matthew Waters committed
384

385
  gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
Matthew Waters's avatar
Matthew Waters committed
386 387
  gl->BindBuffer (GL_ARRAY_BUFFER, 0);

388 389
  gl->DisableVertexAttribArray (filter->draw_attr_position_loc);
  gl->DisableVertexAttribArray (filter->draw_attr_texture_loc);
Matthew Waters's avatar
Matthew Waters committed
390 391 392 393 394
}

static void
_bind_buffer (GstGLOverlay * overlay, GLuint vbo)
{
395
  GstGLFilter *filter = GST_GL_FILTER (overlay);
396
  const GstGLFuncs *gl = GST_GL_BASE_FILTER (overlay)->context->gl_vtable;
Matthew Waters's avatar
Matthew Waters committed
397

398
  gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->vbo_indices);
Matthew Waters's avatar
Matthew Waters committed
399 400
  gl->BindBuffer (GL_ARRAY_BUFFER, vbo);

401 402
  gl->EnableVertexAttribArray (filter->draw_attr_position_loc);
  gl->EnableVertexAttribArray (filter->draw_attr_texture_loc);
Matthew Waters's avatar
Matthew Waters committed
403

404
  gl->VertexAttribPointer (filter->draw_attr_position_loc, 3, GL_FLOAT,
Matthew Waters's avatar
Matthew Waters committed
405
      GL_FALSE, 5 * sizeof (GLfloat), (void *) 0);
406
  gl->VertexAttribPointer (filter->draw_attr_texture_loc, 2, GL_FLOAT,
Matthew Waters's avatar
Matthew Waters committed
407 408 409 410 411 412 413 414 415 416 417
      GL_FALSE, 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
}

/* *INDENT-OFF* */
float v_vertices[] = {
/*|      Vertex     | TexCoord  |*/
  -1.0f, -1.0f, 0.0f, 0.0f, 0.0f,
   1.0f, -1.0f, 0.0f, 1.0f, 0.0f,
   1.0f,  1.0f, 0.0f, 1.0f, 1.0f,
  -1.0f,  1.0f, 0.0f, 0.0f, 1.0f,
};
418 419

static const GLushort indices[] = { 0, 1, 2, 0, 2, 3, };
Matthew Waters's avatar
Matthew Waters committed
420 421
/* *INDENT-ON* */

422 423 424
static gboolean
gst_gl_overlay_callback (GstGLFilter * filter, GstGLMemory * in_tex,
    gpointer stuff)
425
{
426
  GstGLOverlay *overlay = GST_GL_OVERLAY (filter);
427 428
  GstMapInfo map_info;
  guint image_tex;
Matthew Waters's avatar
Matthew Waters committed
429
  gboolean memory_mapped = FALSE;
430
  const GstGLFuncs *gl = GST_GL_BASE_FILTER (filter)->context->gl_vtable;
431
  gboolean ret = FALSE;
Matthew Waters's avatar
Matthew Waters committed
432 433

#if GST_GL_HAVE_OPENGL
434 435
  if (gst_gl_context_get_gl_api (GST_GL_BASE_FILTER (filter)->context) &
      GST_GL_API_OPENGL) {
436

437
    gl->MatrixMode (GL_PROJECTION);
438 439
    gl->LoadIdentity ();
  }
440
#endif
441

Matthew Waters's avatar
Matthew Waters committed
442
  gl->ActiveTexture (GL_TEXTURE0);
443
  gl->BindTexture (GL_TEXTURE_2D, gst_gl_memory_get_texture_id (in_tex));
Matthew Waters's avatar
Matthew Waters committed
444 445

  gst_gl_shader_use (overlay->shader);
446

Matthew Waters's avatar
Matthew Waters committed
447 448 449
  gst_gl_shader_set_uniform_1f (overlay->shader, "alpha", 1.0f);
  gst_gl_shader_set_uniform_1i (overlay->shader, "texture", 0);

450
  filter->draw_attr_position_loc =
Matthew Waters's avatar
Matthew Waters committed
451
      gst_gl_shader_get_attribute_location (overlay->shader, "a_position");
452
  filter->draw_attr_texture_loc =
Matthew Waters's avatar
Matthew Waters committed
453 454
      gst_gl_shader_get_attribute_location (overlay->shader, "a_texcoord");

455
  gst_gl_filter_draw_fullscreen_quad (filter);
Matthew Waters's avatar
Matthew Waters committed
456 457 458

  if (!overlay->image_memory)
    goto out;
459 460 461

  if (!gst_memory_map ((GstMemory *) overlay->image_memory, &map_info,
          GST_MAP_READ | GST_MAP_GL) || map_info.data == NULL)
Matthew Waters's avatar
Matthew Waters committed
462
    goto out;
463

Matthew Waters's avatar
Matthew Waters committed
464
  memory_mapped = TRUE;
465 466
  image_tex = *(guint *) map_info.data;

Matthew Waters's avatar
Matthew Waters committed
467 468 469 470 471 472
  if (!overlay->overlay_vbo) {
    if (gl->GenVertexArrays) {
      gl->GenVertexArrays (1, &overlay->overlay_vao);
      gl->BindVertexArray (overlay->overlay_vao);
    }

473 474 475 476 477
    gl->GenBuffers (1, &overlay->vbo_indices);
    gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->vbo_indices);
    gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
        GL_STATIC_DRAW);

Matthew Waters's avatar
Matthew Waters committed
478 479
    gl->GenBuffers (1, &overlay->overlay_vbo);
    gl->BindBuffer (GL_ARRAY_BUFFER, overlay->overlay_vbo);
480
    gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->vbo_indices);
Matthew Waters's avatar
Matthew Waters committed
481 482 483
    overlay->geometry_change = TRUE;
  }

484 485 486 487
  if (gl->GenVertexArrays) {
    gl->BindVertexArray (overlay->overlay_vao);
  }

Matthew Waters's avatar
Matthew Waters committed
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
  if (overlay->geometry_change) {
    gint render_width, render_height;
    gfloat x, y, image_width, image_height;

    /* *INDENT-OFF* */
    float vertices[] = {
     -1.0f, -1.0f, 0.0f, 0.0f, 0.0f,
      1.0f, -1.0f, 0.0f, 1.0f, 0.0f,
      1.0f,  1.0f, 0.0f, 1.0f, 1.0f,
     -1.0f,  1.0f, 0.0f, 0.0,  1.0f,
    };
    /* *INDENT-ON* */

    /* scale from [0, 1] -> [-1, 1] */
    x = ((gfloat) overlay->offset_x / (gfloat) overlay->window_width +
        overlay->relative_x) * 2.0f - 1.0;
    y = ((gfloat) overlay->offset_y / (gfloat) overlay->window_height +
        overlay->relative_y) * 2.0f - 1.0;
    /* scale from [0, 1] -> [0, 2] */
    render_width =
        overlay->overlay_width >
        0 ? overlay->overlay_width : overlay->image_width;
    render_height =
        overlay->overlay_height >
        0 ? overlay->overlay_height : overlay->image_height;
    image_width =
        ((gfloat) render_width / (gfloat) overlay->window_width) * 2.0f;
    image_height =
        ((gfloat) render_height / (gfloat) overlay->window_height) * 2.0f;

    vertices[0] = vertices[15] = x;
    vertices[5] = vertices[10] = x + image_width;
    vertices[1] = vertices[6] = y;
    vertices[11] = vertices[16] = y + image_height;

    gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices,
        GL_STATIC_DRAW);
  }

527
  _bind_buffer (overlay, overlay->overlay_vbo);
Matthew Waters's avatar
Matthew Waters committed
528 529 530 531

  gl->BindTexture (GL_TEXTURE_2D, image_tex);
  gst_gl_shader_set_uniform_1f (overlay->shader, "alpha", overlay->alpha);

532 533 534 535 536 537 538
  gl->Enable (GL_BLEND);
  gl->BlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
  gl->BlendEquation (GL_FUNC_ADD);

  gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);

  gl->Disable (GL_BLEND);
539
  ret = TRUE;
Matthew Waters's avatar
Matthew Waters committed
540 541

out:
542
  if (gl->GenVertexArrays)
Matthew Waters's avatar
Matthew Waters committed
543
    gl->BindVertexArray (0);
544
  _unbind_buffer (overlay);
Matthew Waters's avatar
Matthew Waters committed
545

546
  gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
Matthew Waters's avatar
Matthew Waters committed
547 548 549

  if (memory_mapped)
    gst_memory_unmap ((GstMemory *) overlay->image_memory, &map_info);
550

Matthew Waters's avatar
Matthew Waters committed
551
  overlay->geometry_change = FALSE;
552 553

  return ret;
554 555
}

556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609
static gboolean
load_file (GstGLOverlay * overlay)
{
  FILE *fp;
  guint8 buff[16];
  gsize n_read;
  GstCaps *caps;
  GstStructure *structure;
  gboolean success = FALSE;

  if (overlay->location == NULL)
    return TRUE;

  if ((fp = fopen (overlay->location, "rb")) == NULL) {
    GST_ELEMENT_ERROR (overlay, RESOURCE, NOT_FOUND, ("Can't open file"),
        ("File: %s", overlay->location));
    return FALSE;
  }

  n_read = fread (buff, 1, sizeof (buff), fp);
  if (n_read != sizeof (buff)) {
    GST_ELEMENT_ERROR (overlay, STREAM, DECODE, ("Can't read file header"),
        ("File: %s", overlay->location));
    goto out;
  }

  caps = gst_type_find_helper_for_data (GST_OBJECT (overlay), buff,
      sizeof (buff), NULL);

  if (caps == NULL) {
    GST_ELEMENT_ERROR (overlay, STREAM, DECODE, ("Can't find file type"),
        ("File: %s", overlay->location));
    goto out;
  }

  fseek (fp, 0, SEEK_SET);

  structure = gst_caps_get_structure (caps, 0);
  if (gst_structure_has_name (structure, "image/jpeg")) {
    success = gst_gl_overlay_load_jpeg (overlay, fp);
  } else if (gst_structure_has_name (structure, "image/png")) {
    success = gst_gl_overlay_load_png (overlay, fp);
  } else {
    GST_ELEMENT_ERROR (overlay, STREAM, DECODE, ("Image type not supported"),
        ("File: %s", overlay->location));
  }

out:
  fclose (fp);
  gst_caps_replace (&caps, NULL);

  return success;
}

610
static gboolean
611 612
gst_gl_overlay_filter_texture (GstGLFilter * filter, GstGLMemory * in_tex,
    GstGLMemory * out_tex)
613 614 615
{
  GstGLOverlay *overlay = GST_GL_OVERLAY (filter);

616
  if (overlay->location_has_changed) {
617 618 619 620
    if (overlay->image_memory) {
      gst_memory_unref ((GstMemory *) overlay->image_memory);
      overlay->image_memory = NULL;
    }
621 622 623

    if (!load_file (overlay))
      return FALSE;
624

625
    overlay->location_has_changed = FALSE;
626 627
  }

628
  gst_gl_filter_render_to_target (filter, in_tex, out_tex,
629 630 631 632 633
      gst_gl_overlay_callback, overlay);

  return TRUE;
}

634 635 636 637 638 639 640 641 642 643 644 645
static void
gst_gl_overlay_before_transform (GstBaseTransform * trans, GstBuffer * outbuf)
{
  GstClockTime stream_time;

  stream_time = gst_segment_to_stream_time (&trans->segment, GST_FORMAT_TIME,
      GST_BUFFER_TIMESTAMP (outbuf));

  if (GST_CLOCK_TIME_IS_VALID (stream_time))
    gst_object_sync_values (GST_OBJECT (trans), stream_time);
}

646 647 648 649 650 651
static void
user_warning_fn (png_structp png_ptr, png_const_charp warning_msg)
{
  g_warning ("%s\n", warning_msg);
}

652
static gboolean
653
gst_gl_overlay_load_jpeg (GstGLOverlay * overlay, FILE * fp)
654
{
655 656
  GstGLBaseMemoryAllocator *mem_allocator;
  GstGLVideoAllocationParams *params;
657
  GstVideoInfo v_info;
658
  GstVideoAlignment v_align;
659
  GstMapInfo map_info;
660 661 662 663 664 665 666 667 668 669
  struct jpeg_decompress_struct cinfo;
  struct jpeg_error_mgr jerr;
  JSAMPROW j;
  int i;

  jpeg_create_decompress (&cinfo);
  cinfo.err = jpeg_std_error (&jerr);
  jpeg_stdio_src (&cinfo, fp);
  jpeg_read_header (&cinfo, TRUE);
  jpeg_start_decompress (&cinfo);
670 671 672
  overlay->image_width = cinfo.image_width;
  overlay->image_height = cinfo.image_height;

673
  if (cinfo.num_components == 1)
674 675
    gst_video_info_set_format (&v_info, GST_VIDEO_FORMAT_Y444,
        overlay->image_width, overlay->image_height);
676
  else
677 678 679
    gst_video_info_set_format (&v_info, GST_VIDEO_FORMAT_RGB,
        overlay->image_width, overlay->image_height);

680 681 682 683
  gst_video_alignment_reset (&v_align);
  v_align.stride_align[0] = 32 - 1;
  gst_video_info_align (&v_info, &v_align);

684
  mem_allocator =
685 686
      GST_GL_BASE_MEMORY_ALLOCATOR (gst_gl_memory_allocator_get_default
      (GST_GL_BASE_FILTER (overlay)->context));
687 688
  params =
      gst_gl_video_allocation_params_new (GST_GL_BASE_FILTER (overlay)->context,
689
      NULL, &v_info, 0, &v_align, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA);
690
  overlay->image_memory = (GstGLMemory *)
691 692 693 694
      gst_gl_base_memory_alloc (mem_allocator,
      (GstGLAllocationParams *) params);
  gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
  gst_object_unref (mem_allocator);
695 696 697

  if (!gst_memory_map ((GstMemory *) overlay->image_memory, &map_info,
          GST_MAP_WRITE)) {
698 699 700
    GST_ELEMENT_ERROR (overlay, STREAM, DECODE, ("failed to map memory"),
        ("File: %s", overlay->location));
    return FALSE;
701 702 703
  }

  for (i = 0; i < overlay->image_height; ++i) {
704
    j = map_info.data + v_info.stride[0] * i;
705 706 707 708
    jpeg_read_scanlines (&cinfo, &j, 1);
  }
  jpeg_finish_decompress (&cinfo);
  jpeg_destroy_decompress (&cinfo);
709
  gst_memory_unmap ((GstMemory *) overlay->image_memory, &map_info);
710

711
  return TRUE;
712 713
}

714
static gboolean
715
gst_gl_overlay_load_png (GstGLOverlay * overlay, FILE * fp)
716
{
717 718
  GstGLBaseMemoryAllocator *mem_allocator;
  GstGLVideoAllocationParams *params;
719 720
  GstVideoInfo v_info;
  GstMapInfo map_info;
721 722 723 724 725 726 727 728 729 730 731 732 733 734

  png_structp png_ptr;
  png_infop info_ptr;
  png_uint_32 width = 0;
  png_uint_32 height = 0;
  gint bit_depth = 0;
  gint color_type = 0;
  gint interlace_type = 0;
  guint y = 0;
  guchar **rows = NULL;
  gint filler;
  png_byte magic[8];
  gint n_read;

735
  if (!GST_GL_BASE_FILTER (overlay)->context)
736
    return FALSE;
737 738 739 740

  /* Read magic number */
  n_read = fread (magic, 1, sizeof (magic), fp);
  if (n_read != sizeof (magic)) {
741 742 743
    GST_ELEMENT_ERROR (overlay, STREAM, DECODE,
        ("can't read PNG magic number"), ("File: %s", overlay->location));
    return FALSE;
744 745 746 747
  }

  /* Check for valid magic number */
  if (png_sig_cmp (magic, 0, sizeof (magic))) {
748 749 750
    GST_ELEMENT_ERROR (overlay, STREAM, DECODE,
        ("not a valid PNG image"), ("File: %s", overlay->location));
    return FALSE;
751 752 753 754 755
  }

  png_ptr = png_create_read_struct (PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);

  if (png_ptr == NULL) {
756 757 758 759
    GST_ELEMENT_ERROR (overlay, STREAM, DECODE,
        ("failed to initialize the png_struct"), ("File: %s",
            overlay->location));
    return FALSE;
760 761 762 763 764 765 766
  }

  png_set_error_fn (png_ptr, NULL, NULL, user_warning_fn);

  info_ptr = png_create_info_struct (png_ptr);
  if (info_ptr == NULL) {
    png_destroy_read_struct (&png_ptr, png_infopp_NULL, png_infopp_NULL);
767 768 769 770
    GST_ELEMENT_ERROR (overlay, STREAM, DECODE,
        ("failed to initialize the memory for image information"),
        ("File: %s", overlay->location));
    return FALSE;
771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789
  }

  png_init_io (png_ptr, fp);

  png_set_sig_bytes (png_ptr, sizeof (magic));

  png_read_info (png_ptr, info_ptr);

  png_get_IHDR (png_ptr, info_ptr, &width, &height, &bit_depth, &color_type,
      &interlace_type, int_p_NULL, int_p_NULL);

  if (color_type == PNG_COLOR_TYPE_RGB) {
    filler = 0xff;
    png_set_filler (png_ptr, filler, PNG_FILLER_AFTER);
    color_type = PNG_COLOR_TYPE_RGB_ALPHA;
  }

  if (color_type != PNG_COLOR_TYPE_RGB_ALPHA) {
    png_destroy_read_struct (&png_ptr, png_infopp_NULL, png_infopp_NULL);
790 791 792
    GST_ELEMENT_ERROR (overlay, STREAM, DECODE,
        ("color type is not rgb"), ("File: %s", overlay->location));
    return FALSE;
793 794
  }

795 796
  overlay->image_width = width;
  overlay->image_height = height;
797

798
  gst_video_info_set_format (&v_info, GST_VIDEO_FORMAT_RGBA, width, height);
799
  mem_allocator =
800 801
      GST_GL_BASE_MEMORY_ALLOCATOR (gst_gl_memory_allocator_get_default
      (GST_GL_BASE_FILTER (overlay)->context));
802 803
  params =
      gst_gl_video_allocation_params_new (GST_GL_BASE_FILTER (overlay)->context,
804
      NULL, &v_info, 0, NULL, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA);
805
  overlay->image_memory = (GstGLMemory *)
806 807 808 809
      gst_gl_base_memory_alloc (mem_allocator,
      (GstGLAllocationParams *) params);
  gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
  gst_object_unref (mem_allocator);
810

811 812
  if (!gst_memory_map ((GstMemory *) overlay->image_memory, &map_info,
          GST_MAP_WRITE)) {
Xavier Claessens's avatar
Xavier Claessens committed
813
    png_destroy_read_struct (&png_ptr, &info_ptr, png_infopp_NULL);
814 815 816
    GST_ELEMENT_ERROR (overlay, STREAM, DECODE,
        ("failed to map memory"), ("File: %s", overlay->location));
    return FALSE;
817
  }
818 819 820
  rows = (guchar **) malloc (sizeof (guchar *) * height);

  for (y = 0; y < height; ++y)
821
    rows[y] = (guchar *) (map_info.data + y * width * 4);
822 823 824 825

  png_read_image (png_ptr, rows);

  free (rows);
826
  gst_memory_unmap ((GstMemory *) overlay->image_memory, &map_info);
827 828 829 830

  png_read_end (png_ptr, info_ptr);
  png_destroy_read_struct (&png_ptr, &info_ptr, png_infopp_NULL);

831
  return TRUE;
832
}