gstglfiltershader.c 17.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * glshader gstreamer plugin
 * Copyrithg (C) 2008 Filippo Argiolas <filippo.argiolas@gmail.com>
 * Copyright (C) 2009 Luc Deschenaux <luc.deschenaux@freesurf.ch>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/**
 * SECTION:element-glshader
24
 * @title: glshader
25
 *
Florent Thiery's avatar
Florent Thiery committed
26
 * OpenGL fragment shader filter
27
 *
28
 * ## Examples
29
 * |[
Florent Thiery's avatar
Florent Thiery committed
30
 * gst-launch-1.0 videotestsrc ! glupload ! glshader fragment="\"`cat myshader.frag`\"" ! glimagesink
31
 * ]|
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
 * FBO (Frame Buffer Object) and GLSL (OpenGL Shading Language) are required.
 * Depending on the exact OpenGL version chosen and the exact requirements of
 * the OpenGL implementation, a #version header may be required.
 *
 * The following is a simple OpenGL ES (also usable with OpenGL 3 core contexts)
 * passthrough shader with the required inputs.
 * |[
 * #version 100
 * #ifdef GL_ES
 * precision mediump float;
 * #endif
 * varying vec2 v_texcoord;
 * uniform sampler2D tex;
 * uniform float time;
 * uniform float width;
 * uniform float height;
48
 *
49 50 51 52 53
 * void main () {
 *   gl_FragColor = texture2D( tex, v_texcoord );
 * }
 * ]|
 *
54 55 56 57 58
 */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

59
#include <gst/gl/gstglfuncs.h>
60 61

#include "gstglfiltershader.h"
62 63 64
#if HAVE_GRAPHENE
#include <graphene-gobject.h>
#endif
65 66 67 68

enum
{
  PROP_0,
69 70 71
  PROP_SHADER,
  PROP_VERTEX,
  PROP_FRAGMENT,
72
  PROP_UNIFORMS,
73
  PROP_UPDATE_SHADER,
74 75 76 77 78 79 80 81
  PROP_LAST,
};

enum
{
  SIGNAL_0,
  SIGNAL_CREATE_SHADER,
  SIGNAL_LAST,
82 83
};

84
static guint gst_gl_shader_signals[SIGNAL_LAST] = { 0 };
85

86 87 88 89 90
#define GST_CAT_DEFAULT gst_gl_filtershader_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);

#define DEBUG_INIT \
  GST_DEBUG_CATEGORY_INIT (gst_gl_filtershader_debug, "glshader", 0, "glshader element");
91
#define gst_gl_filtershader_parent_class parent_class
92 93 94
G_DEFINE_TYPE_WITH_CODE (GstGLFilterShader, gst_gl_filtershader,
    GST_TYPE_GL_FILTER, DEBUG_INIT);

95
static void gst_gl_filtershader_finalize (GObject * object);
96 97 98 99
static void gst_gl_filtershader_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_gl_filtershader_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);
100 101
static gboolean gst_gl_filtershader_gl_start (GstGLBaseFilter * base);
static void gst_gl_filtershader_gl_stop (GstGLBaseFilter * base);
102 103
static gboolean gst_gl_filtershader_filter (GstGLFilter * filter,
    GstBuffer * inbuf, GstBuffer * outbuf);
104
static gboolean gst_gl_filtershader_filter_texture (GstGLFilter * filter,
105
    GstGLMemory * in_tex, GstGLMemory * out_tex);
106 107
static gboolean gst_gl_filtershader_hcallback (GstGLFilter * filter,
    GstGLMemory * in_tex, gpointer stuff);
108 109 110 111 112 113 114 115 116 117

static void
gst_gl_filtershader_class_init (GstGLFilterShaderClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *element_class;

  gobject_class = (GObjectClass *) klass;
  element_class = GST_ELEMENT_CLASS (klass);

118 119
  gst_gl_filter_add_rgba_pad_templates (GST_GL_FILTER_CLASS (klass));

120
  gobject_class->finalize = gst_gl_filtershader_finalize;
121 122 123
  gobject_class->set_property = gst_gl_filtershader_set_property;
  gobject_class->get_property = gst_gl_filtershader_get_property;

124 125
  g_object_class_install_property (gobject_class, PROP_SHADER,
      g_param_spec_object ("shader", "Shader object",
126
          "GstGLShader to use", GST_TYPE_GL_SHADER,
127 128
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

129 130 131
  g_object_class_install_property (gobject_class, PROP_VERTEX,
      g_param_spec_string ("vertex", "Vertex Source",
          "GLSL vertex source", NULL,
132 133
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

134 135 136 137 138 139
  g_object_class_install_property (gobject_class, PROP_FRAGMENT,
      g_param_spec_string ("fragment", "Fragment Source",
          "GLSL fragment source", NULL,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  /* FIXME: add other stages */

140 141 142 143 144
  g_object_class_install_property (gobject_class, PROP_UNIFORMS,
      g_param_spec_boxed ("uniforms", "GLSL Uniforms",
          "GLSL Uniforms", GST_TYPE_STRUCTURE,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

145 146 147 148 149 150 151 152 153 154 155 156 157 158
  g_object_class_install_property (gobject_class, PROP_UPDATE_SHADER,
      g_param_spec_boolean ("update-shader", "Update Shader",
          "Emit the \'create-shader\' signal for the next frame",
          FALSE, G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS));

  /*
   * GstGLFilterShader::create-shader:
   * @object: the #GstGLFilterShader
   *
   * Ask's the application for a shader to render with as a result of
   * inititialization or setting the 'update-shader' property.
   *
   * Returns: a new #GstGLShader for use in the rendering pipeline
   */
159 160 161
  gst_gl_shader_signals[SIGNAL_CREATE_SHADER] =
      g_signal_new ("create-shader", G_TYPE_FROM_CLASS (klass),
      G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
162
      GST_TYPE_GL_SHADER, 0);
163 164 165

  gst_element_class_set_metadata (element_class,
      "OpenGL fragment shader filter", "Filter/Effect",
166
      "Perform operations with a GLSL shader", "<matthew@centricular.com>");
167

168
  GST_GL_FILTER_CLASS (klass)->filter = gst_gl_filtershader_filter;
169 170
  GST_GL_FILTER_CLASS (klass)->filter_texture =
      gst_gl_filtershader_filter_texture;
171

172 173
  GST_GL_BASE_FILTER_CLASS (klass)->gl_start = gst_gl_filtershader_gl_start;
  GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = gst_gl_filtershader_gl_stop;
174
  GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
175
      GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3;
176 177 178 179 180
}

static void
gst_gl_filtershader_init (GstGLFilterShader * filtershader)
{
181
  filtershader->new_source = TRUE;
182 183
}

184 185
static void
gst_gl_filtershader_finalize (GObject * object)
186
{
187 188
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (object);

189
  g_free (filtershader->vertex);
190 191
  filtershader->vertex = NULL;

192
  g_free (filtershader->fragment);
193 194
  filtershader->fragment = NULL;

195 196 197 198
  if (filtershader->uniforms)
    gst_structure_free (filtershader->uniforms);
  filtershader->uniforms = NULL;

199
  G_OBJECT_CLASS (gst_gl_filtershader_parent_class)->finalize (object);
200 201 202 203 204 205 206 207 208
}

static void
gst_gl_filtershader_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (object);

  switch (prop_id) {
209 210
    case PROP_SHADER:
      GST_OBJECT_LOCK (filtershader);
211
      gst_object_replace ((GstObject **) & filtershader->shader,
212
          g_value_dup_object (value));
213
      filtershader->new_source = FALSE;
214
      GST_OBJECT_UNLOCK (filtershader);
215
      break;
216 217
    case PROP_VERTEX:
      GST_OBJECT_LOCK (filtershader);
218
      g_free (filtershader->vertex);
219 220 221
      filtershader->vertex = g_value_dup_string (value);
      filtershader->new_source = TRUE;
      GST_OBJECT_UNLOCK (filtershader);
222
      break;
223 224
    case PROP_FRAGMENT:
      GST_OBJECT_LOCK (filtershader);
225
      g_free (filtershader->fragment);
226 227 228
      filtershader->fragment = g_value_dup_string (value);
      filtershader->new_source = TRUE;
      GST_OBJECT_UNLOCK (filtershader);
229
      break;
230 231 232 233 234 235 236 237
    case PROP_UNIFORMS:
      GST_OBJECT_LOCK (filtershader);
      if (filtershader->uniforms)
        gst_structure_free (filtershader->uniforms);
      filtershader->uniforms = g_value_dup_boxed (value);
      filtershader->new_uniforms = TRUE;
      GST_OBJECT_UNLOCK (filtershader);
      break;
238 239 240 241 242
    case PROP_UPDATE_SHADER:
      GST_OBJECT_LOCK (filtershader);
      filtershader->update_shader = g_value_get_boolean (value);
      GST_OBJECT_UNLOCK (filtershader);
      break;
243 244 245 246 247 248 249 250 251 252 253 254 255
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_gl_filtershader_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (object);

  switch (prop_id) {
256 257
    case PROP_SHADER:
      GST_OBJECT_LOCK (filtershader);
258
      g_value_set_object (value, filtershader->shader);
259
      GST_OBJECT_UNLOCK (filtershader);
260
      break;
261 262 263 264 265 266 267 268 269
    case PROP_VERTEX:
      GST_OBJECT_LOCK (filtershader);
      g_value_set_string (value, filtershader->vertex);
      GST_OBJECT_UNLOCK (filtershader);
      break;
    case PROP_FRAGMENT:
      GST_OBJECT_LOCK (filtershader);
      g_value_set_string (value, filtershader->fragment);
      GST_OBJECT_UNLOCK (filtershader);
270
      break;
271 272 273 274 275
    case PROP_UNIFORMS:
      GST_OBJECT_LOCK (filtershader);
      g_value_set_boxed (value, filtershader->uniforms);
      GST_OBJECT_UNLOCK (filtershader);
      break;
276 277 278 279 280 281
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

282 283
static void
gst_gl_filtershader_gl_stop (GstGLBaseFilter * base)
284
{
285
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (base);
286

287 288 289
  if (filtershader->shader)
    gst_object_unref (filtershader->shader);
  filtershader->shader = NULL;
290

291
  GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base);
292 293 294
}

static gboolean
295
gst_gl_filtershader_gl_start (GstGLBaseFilter * base)
296
{
297
  return GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base);
298 299
}

300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
static inline gboolean
_gst_clock_time_to_double (GstClockTime time, gdouble * result)
{
  if (!GST_CLOCK_TIME_IS_VALID (time))
    return FALSE;

  *result = (gdouble) time / GST_SECOND;

  return TRUE;
}

static inline gboolean
_gint64_time_val_to_double (gint64 time, gdouble * result)
{
  if (time == -1)
    return FALSE;

  *result = (gdouble) time / GST_USECOND;

  return TRUE;
}

static gboolean
gst_gl_filtershader_filter (GstGLFilter * filter, GstBuffer * inbuf,
    GstBuffer * outbuf)
{
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (filter);

  if (!_gst_clock_time_to_double (GST_BUFFER_PTS (inbuf), &filtershader->time)) {
    if (!_gst_clock_time_to_double (GST_BUFFER_DTS (inbuf),
            &filtershader->time))
      _gint64_time_val_to_double (g_get_monotonic_time (), &filtershader->time);
  }

  return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
}

337
static gboolean
338 339
gst_gl_filtershader_filter_texture (GstGLFilter * filter, GstGLMemory * in_tex,
    GstGLMemory * out_tex)
340 341 342
{
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (filter);

343 344
  gst_gl_filter_render_to_target (filter, in_tex, out_tex,
      gst_gl_filtershader_hcallback, NULL);
345

346 347 348
  if (!filtershader->shader)
    return FALSE;

349 350 351
  return TRUE;
}

352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
static gboolean
_set_uniform (GQuark field_id, const GValue * value, gpointer user_data)
{
  GstGLShader *shader = user_data;
  const gchar *field_name = g_quark_to_string (field_id);

  if (G_TYPE_CHECK_VALUE_TYPE ((value), G_TYPE_INT)) {
    gst_gl_shader_set_uniform_1i (shader, field_name, g_value_get_int (value));
  } else if (G_TYPE_CHECK_VALUE_TYPE ((value), G_TYPE_FLOAT)) {
    gst_gl_shader_set_uniform_1f (shader, field_name,
        g_value_get_float (value));
#if HAVE_GRAPHENE
  } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_VEC2)) {
    graphene_vec2_t *vec2 = g_value_get_boxed (value);
    float x = graphene_vec2_get_x (vec2);
    float y = graphene_vec2_get_y (vec2);
    gst_gl_shader_set_uniform_2f (shader, field_name, x, y);
  } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_VEC3)) {
    graphene_vec3_t *vec3 = g_value_get_boxed (value);
    float x = graphene_vec3_get_x (vec3);
    float y = graphene_vec3_get_y (vec3);
    float z = graphene_vec3_get_z (vec3);
    gst_gl_shader_set_uniform_3f (shader, field_name, x, y, z);
  } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_VEC4)) {
    graphene_vec4_t *vec4 = g_value_get_boxed (value);
    float x = graphene_vec4_get_x (vec4);
    float y = graphene_vec4_get_y (vec4);
    float z = graphene_vec4_get_z (vec4);
    float w = graphene_vec4_get_w (vec4);
    gst_gl_shader_set_uniform_4f (shader, field_name, x, y, z, w);
  } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_MATRIX)) {
    graphene_matrix_t *matrix = g_value_get_boxed (value);
    float matrix_f[16];
    graphene_matrix_to_float (matrix, matrix_f);
    gst_gl_shader_set_uniform_matrix_4fv (shader, field_name, 1, FALSE,
        matrix_f);
#endif
  } else {
    /* FIXME: Add support for unsigned ints, non 4x4 matrices, etc */
    GST_FIXME ("Don't know how to set the \'%s\' paramater.  Unknown type",
        field_name);
    return TRUE;
  }

  return TRUE;
}

static void
_update_uniforms (GstGLFilterShader * filtershader)
{
  if (filtershader->new_uniforms && filtershader->uniforms) {
    gst_gl_shader_use (filtershader->shader);

    gst_structure_foreach (filtershader->uniforms,
        (GstStructureForeachFunc) _set_uniform, filtershader->shader);
    filtershader->new_uniforms = FALSE;
  }
}

411 412 413 414 415 416 417 418 419
static GstGLShader *
_maybe_recompile_shader (GstGLFilterShader * filtershader)
{
  GstGLContext *context = GST_GL_BASE_FILTER (filtershader)->context;
  GstGLShader *shader;
  GError *error = NULL;

  GST_OBJECT_LOCK (filtershader);

420 421
  if (!filtershader->shader || filtershader->update_shader) {
    filtershader->update_shader = FALSE;
422 423
    GST_OBJECT_UNLOCK (filtershader);
    g_signal_emit (filtershader, gst_gl_shader_signals[SIGNAL_CREATE_SHADER], 0,
424 425 426 427 428 429
        &shader);
    GST_OBJECT_LOCK (filtershader);

    if (shader) {
      if (filtershader->shader)
        gst_object_unref (filtershader->shader);
430
      filtershader->new_source = FALSE;
431
      filtershader->shader = gst_object_ref (shader);
432 433
      filtershader->new_uniforms = TRUE;
      _update_uniforms (filtershader);
434 435 436 437 438
      GST_OBJECT_UNLOCK (filtershader);
      return shader;
    }
  }

439 440
  if (filtershader->shader) {
    shader = gst_object_ref (filtershader->shader);
441
    _update_uniforms (filtershader);
442 443 444 445
    GST_OBJECT_UNLOCK (filtershader);
    return shader;
  }

446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
  if (filtershader->new_source) {
    GstGLSLStage *stage;

    shader = gst_gl_shader_new (context);

    if (filtershader->vertex) {
      if (!(stage = gst_glsl_stage_new_with_string (context, GL_VERTEX_SHADER,
                  GST_GLSL_VERSION_NONE, GST_GLSL_PROFILE_NONE,
                  filtershader->vertex))) {
        g_set_error (&error, GST_GLSL_ERROR, GST_GLSL_ERROR_COMPILE,
            "Failed to create shader vertex stage");
        goto print_error;
      }
    } else {
      stage = gst_glsl_stage_new_default_vertex (context);
    }

    if (!gst_gl_shader_compile_attach_stage (shader, stage, &error)) {
      gst_object_unref (stage);
      goto print_error;
    }

    if (filtershader->fragment) {
      if (!(stage = gst_glsl_stage_new_with_string (context, GL_FRAGMENT_SHADER,
                  GST_GLSL_VERSION_NONE, GST_GLSL_PROFILE_NONE,
                  filtershader->fragment))) {
        g_set_error (&error, GST_GLSL_ERROR, GST_GLSL_ERROR_COMPILE,
            "Failed to create shader fragment stage");
        goto print_error;
      }
    } else {
      stage = gst_glsl_stage_new_default_fragment (context);
    }

    if (!gst_gl_shader_compile_attach_stage (shader, stage, &error)) {
      gst_object_unref (stage);
      goto print_error;
    }

    if (!gst_gl_shader_link (shader, &error)) {
      goto print_error;
    }
    if (filtershader->shader)
      gst_object_unref (filtershader->shader);
    filtershader->shader = gst_object_ref (shader);
    filtershader->new_source = FALSE;
492 493
    filtershader->new_uniforms = TRUE;
    _update_uniforms (filtershader);
494 495 496 497

    GST_OBJECT_UNLOCK (filtershader);
    return shader;
  } else if (filtershader->shader) {
498
    _update_uniforms (filtershader);
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
    shader = gst_object_ref (filtershader->shader);
    GST_OBJECT_UNLOCK (filtershader);
    return shader;
  }

  return NULL;

print_error:
  if (shader) {
    gst_object_unref (shader);
    shader = NULL;
  }

  GST_OBJECT_UNLOCK (filtershader);
  GST_ELEMENT_ERROR (filtershader, RESOURCE, NOT_FOUND,
      ("%s", error->message), (NULL));
  return NULL;
}

518 519
static gboolean
gst_gl_filtershader_hcallback (GstGLFilter * filter, GstGLMemory * in_tex,
520 521 522
    gpointer stuff)
{
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (filter);
523
  GstGLFuncs *gl = GST_GL_BASE_FILTER (filter)->context->gl_vtable;
524
  GstGLShader *shader;
525

526
  if (!(shader = _maybe_recompile_shader (filtershader)))
527
    return FALSE;
528

529 530
  gl->ClearColor (0.0, 0.0, 0.0, 1.0);
  gl->Clear (GL_COLOR_BUFFER_BIT);
531

532
  gst_gl_shader_use (shader);
533

534 535
  /* FIXME: propertise these */
  gst_gl_shader_set_uniform_1i (shader, "tex", 0);
536 537 538 539
  gst_gl_shader_set_uniform_1f (shader, "width",
      GST_VIDEO_INFO_WIDTH (&filter->out_info));
  gst_gl_shader_set_uniform_1f (shader, "height",
      GST_VIDEO_INFO_HEIGHT (&filter->out_info));
540 541 542
  gst_gl_shader_set_uniform_1f (shader, "time", filtershader->time);

  /* FIXME: propertise these */
Matthew Waters's avatar
Matthew Waters committed
543
  filter->draw_attr_position_loc =
544
      gst_gl_shader_get_attribute_location (shader, "a_position");
Matthew Waters's avatar
Matthew Waters committed
545
  filter->draw_attr_texture_loc =
546
      gst_gl_shader_get_attribute_location (shader, "a_texcoord");
547

548
  gl->ActiveTexture (GL_TEXTURE0);
549
  gl->BindTexture (GL_TEXTURE_2D, gst_gl_memory_get_texture_id (in_tex));
550

551
  gst_gl_filter_draw_fullscreen_quad (filter);
552 553

  gst_object_unref (shader);
554 555

  return TRUE;
556
}