gstglfiltershader.c 16.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
/*
 * glshader gstreamer plugin
 * Copyrithg (C) 2008 Filippo Argiolas <filippo.argiolas@gmail.com>
 * Copyright (C) 2009 Luc Deschenaux <luc.deschenaux@freesurf.ch>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/**
 * SECTION:element-glshader
 *
Florent Thiery's avatar
Florent Thiery committed
25
 * OpenGL fragment shader filter
26 27 28 29
 *
 * <refsect2>
 * <title>Examples</title>
 * |[
Florent Thiery's avatar
Florent Thiery committed
30
 * gst-launch-1.0 videotestsrc ! glupload ! glshader fragment="\"`cat myshader.frag`\"" ! glimagesink
31
 * ]|
Florent Thiery's avatar
Florent Thiery committed
32
 * FBO (Frame Buffer Object) and GLSL (OpenGL Shading Language) are required. A #version header is required
33 34 35 36 37 38 39 40 41
 * </refsect2>
 */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <gst/gst.h>

#include "gstglfiltershader.h"
42 43 44
#if HAVE_GRAPHENE
#include <graphene-gobject.h>
#endif
45 46 47 48

enum
{
  PROP_0,
49 50 51
  PROP_SHADER,
  PROP_VERTEX,
  PROP_FRAGMENT,
52
  PROP_UNIFORMS,
53
  PROP_UPDATE_SHADER,
54 55 56 57 58 59 60 61
  PROP_LAST,
};

enum
{
  SIGNAL_0,
  SIGNAL_CREATE_SHADER,
  SIGNAL_LAST,
62 63
};

64
static guint gst_gl_shader_signals[SIGNAL_LAST] = { 0 };
65

66 67 68 69 70
#define GST_CAT_DEFAULT gst_gl_filtershader_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);

#define DEBUG_INIT \
  GST_DEBUG_CATEGORY_INIT (gst_gl_filtershader_debug, "glshader", 0, "glshader element");
71
#define gst_gl_filtershader_parent_class parent_class
72 73 74
G_DEFINE_TYPE_WITH_CODE (GstGLFilterShader, gst_gl_filtershader,
    GST_TYPE_GL_FILTER, DEBUG_INIT);

75
static void gst_gl_filtershader_finalize (GObject * object);
76 77 78 79
static void gst_gl_filtershader_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_gl_filtershader_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);
80 81
static gboolean gst_gl_filtershader_gl_start (GstGLBaseFilter * base);
static void gst_gl_filtershader_gl_stop (GstGLBaseFilter * base);
82 83
static gboolean gst_gl_filtershader_filter (GstGLFilter * filter,
    GstBuffer * inbuf, GstBuffer * outbuf);
84
static gboolean gst_gl_filtershader_filter_texture (GstGLFilter * filter,
85
    GstGLMemory * in_tex, GstGLMemory * out_tex);
86 87
static gboolean gst_gl_filtershader_hcallback (GstGLFilter * filter,
    GstGLMemory * in_tex, gpointer stuff);
88 89 90 91 92 93 94 95 96 97

static void
gst_gl_filtershader_class_init (GstGLFilterShaderClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *element_class;

  gobject_class = (GObjectClass *) klass;
  element_class = GST_ELEMENT_CLASS (klass);

98
  gobject_class->finalize = gst_gl_filtershader_finalize;
99 100 101
  gobject_class->set_property = gst_gl_filtershader_set_property;
  gobject_class->get_property = gst_gl_filtershader_get_property;

102 103 104
  g_object_class_install_property (gobject_class, PROP_SHADER,
      g_param_spec_object ("shader", "Shader object",
          "GstGLShader to use", GST_GL_TYPE_SHADER,
105 106
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

107 108 109
  g_object_class_install_property (gobject_class, PROP_VERTEX,
      g_param_spec_string ("vertex", "Vertex Source",
          "GLSL vertex source", NULL,
110 111
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

112 113 114 115 116 117
  g_object_class_install_property (gobject_class, PROP_FRAGMENT,
      g_param_spec_string ("fragment", "Fragment Source",
          "GLSL fragment source", NULL,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
  /* FIXME: add other stages */

118 119 120 121 122
  g_object_class_install_property (gobject_class, PROP_UNIFORMS,
      g_param_spec_boxed ("uniforms", "GLSL Uniforms",
          "GLSL Uniforms", GST_TYPE_STRUCTURE,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

123 124 125 126 127 128 129 130 131 132 133 134 135 136
  g_object_class_install_property (gobject_class, PROP_UPDATE_SHADER,
      g_param_spec_boolean ("update-shader", "Update Shader",
          "Emit the \'create-shader\' signal for the next frame",
          FALSE, G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS));

  /*
   * GstGLFilterShader::create-shader:
   * @object: the #GstGLFilterShader
   *
   * Ask's the application for a shader to render with as a result of
   * inititialization or setting the 'update-shader' property.
   *
   * Returns: a new #GstGLShader for use in the rendering pipeline
   */
137 138 139 140
  gst_gl_shader_signals[SIGNAL_CREATE_SHADER] =
      g_signal_new ("create-shader", G_TYPE_FROM_CLASS (klass),
      G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
      GST_GL_TYPE_SHADER, 0);
141 142 143

  gst_element_class_set_metadata (element_class,
      "OpenGL fragment shader filter", "Filter/Effect",
144
      "Perform operations with a GLSL shader", "<matthew@centricular.com>");
145

146
  GST_GL_FILTER_CLASS (klass)->filter = gst_gl_filtershader_filter;
147 148
  GST_GL_FILTER_CLASS (klass)->filter_texture =
      gst_gl_filtershader_filter_texture;
149

150 151
  GST_GL_BASE_FILTER_CLASS (klass)->gl_start = gst_gl_filtershader_gl_start;
  GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = gst_gl_filtershader_gl_stop;
152
  GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
153
      GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3;
154 155 156 157 158
}

static void
gst_gl_filtershader_init (GstGLFilterShader * filtershader)
{
159
  filtershader->new_source = TRUE;
160 161
}

162 163
static void
gst_gl_filtershader_finalize (GObject * object)
164
{
165 166
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (object);

167
  g_free (filtershader->vertex);
168 169
  filtershader->vertex = NULL;

170
  g_free (filtershader->fragment);
171 172
  filtershader->fragment = NULL;

173 174 175 176
  if (filtershader->uniforms)
    gst_structure_free (filtershader->uniforms);
  filtershader->uniforms = NULL;

177
  G_OBJECT_CLASS (gst_gl_filtershader_parent_class)->finalize (object);
178 179 180 181 182 183 184 185 186
}

static void
gst_gl_filtershader_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (object);

  switch (prop_id) {
187 188
    case PROP_SHADER:
      GST_OBJECT_LOCK (filtershader);
189
      gst_object_replace ((GstObject **) & filtershader->shader,
190
          g_value_dup_object (value));
191
      filtershader->new_source = FALSE;
192
      GST_OBJECT_UNLOCK (filtershader);
193
      break;
194 195
    case PROP_VERTEX:
      GST_OBJECT_LOCK (filtershader);
196
      g_free (filtershader->vertex);
197 198 199
      filtershader->vertex = g_value_dup_string (value);
      filtershader->new_source = TRUE;
      GST_OBJECT_UNLOCK (filtershader);
200
      break;
201 202
    case PROP_FRAGMENT:
      GST_OBJECT_LOCK (filtershader);
203
      g_free (filtershader->fragment);
204 205 206
      filtershader->fragment = g_value_dup_string (value);
      filtershader->new_source = TRUE;
      GST_OBJECT_UNLOCK (filtershader);
207
      break;
208 209 210 211 212 213 214 215
    case PROP_UNIFORMS:
      GST_OBJECT_LOCK (filtershader);
      if (filtershader->uniforms)
        gst_structure_free (filtershader->uniforms);
      filtershader->uniforms = g_value_dup_boxed (value);
      filtershader->new_uniforms = TRUE;
      GST_OBJECT_UNLOCK (filtershader);
      break;
216 217 218 219 220
    case PROP_UPDATE_SHADER:
      GST_OBJECT_LOCK (filtershader);
      filtershader->update_shader = g_value_get_boolean (value);
      GST_OBJECT_UNLOCK (filtershader);
      break;
221 222 223 224 225 226 227 228 229 230 231 232 233
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_gl_filtershader_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (object);

  switch (prop_id) {
234 235
    case PROP_SHADER:
      GST_OBJECT_LOCK (filtershader);
236
      g_value_set_object (value, filtershader->shader);
237
      GST_OBJECT_UNLOCK (filtershader);
238
      break;
239 240 241 242 243 244 245 246 247
    case PROP_VERTEX:
      GST_OBJECT_LOCK (filtershader);
      g_value_set_string (value, filtershader->vertex);
      GST_OBJECT_UNLOCK (filtershader);
      break;
    case PROP_FRAGMENT:
      GST_OBJECT_LOCK (filtershader);
      g_value_set_string (value, filtershader->fragment);
      GST_OBJECT_UNLOCK (filtershader);
248
      break;
249 250 251 252 253
    case PROP_UNIFORMS:
      GST_OBJECT_LOCK (filtershader);
      g_value_set_boxed (value, filtershader->uniforms);
      GST_OBJECT_UNLOCK (filtershader);
      break;
254 255 256 257 258 259
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

260 261
static void
gst_gl_filtershader_gl_stop (GstGLBaseFilter * base)
262
{
263
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (base);
264

265 266 267
  if (filtershader->shader)
    gst_object_unref (filtershader->shader);
  filtershader->shader = NULL;
268

269
  GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base);
270 271 272
}

static gboolean
273
gst_gl_filtershader_gl_start (GstGLBaseFilter * base)
274
{
275
  return GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base);
276 277
}

278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
static inline gboolean
_gst_clock_time_to_double (GstClockTime time, gdouble * result)
{
  if (!GST_CLOCK_TIME_IS_VALID (time))
    return FALSE;

  *result = (gdouble) time / GST_SECOND;

  return TRUE;
}

static inline gboolean
_gint64_time_val_to_double (gint64 time, gdouble * result)
{
  if (time == -1)
    return FALSE;

  *result = (gdouble) time / GST_USECOND;

  return TRUE;
}

static gboolean
gst_gl_filtershader_filter (GstGLFilter * filter, GstBuffer * inbuf,
    GstBuffer * outbuf)
{
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (filter);

  if (!_gst_clock_time_to_double (GST_BUFFER_PTS (inbuf), &filtershader->time)) {
    if (!_gst_clock_time_to_double (GST_BUFFER_DTS (inbuf),
            &filtershader->time))
      _gint64_time_val_to_double (g_get_monotonic_time (), &filtershader->time);
  }

  return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
}

315
static gboolean
316 317
gst_gl_filtershader_filter_texture (GstGLFilter * filter, GstGLMemory * in_tex,
    GstGLMemory * out_tex)
318 319 320
{
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (filter);

321 322
  gst_gl_filter_render_to_target (filter, in_tex, out_tex,
      gst_gl_filtershader_hcallback, NULL);
323

324 325 326
  if (!filtershader->shader)
    return FALSE;

327 328 329
  return TRUE;
}

330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
static gboolean
_set_uniform (GQuark field_id, const GValue * value, gpointer user_data)
{
  GstGLShader *shader = user_data;
  const gchar *field_name = g_quark_to_string (field_id);

  if (G_TYPE_CHECK_VALUE_TYPE ((value), G_TYPE_INT)) {
    gst_gl_shader_set_uniform_1i (shader, field_name, g_value_get_int (value));
  } else if (G_TYPE_CHECK_VALUE_TYPE ((value), G_TYPE_FLOAT)) {
    gst_gl_shader_set_uniform_1f (shader, field_name,
        g_value_get_float (value));
#if HAVE_GRAPHENE
  } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_VEC2)) {
    graphene_vec2_t *vec2 = g_value_get_boxed (value);
    float x = graphene_vec2_get_x (vec2);
    float y = graphene_vec2_get_y (vec2);
    gst_gl_shader_set_uniform_2f (shader, field_name, x, y);
  } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_VEC3)) {
    graphene_vec3_t *vec3 = g_value_get_boxed (value);
    float x = graphene_vec3_get_x (vec3);
    float y = graphene_vec3_get_y (vec3);
    float z = graphene_vec3_get_z (vec3);
    gst_gl_shader_set_uniform_3f (shader, field_name, x, y, z);
  } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_VEC4)) {
    graphene_vec4_t *vec4 = g_value_get_boxed (value);
    float x = graphene_vec4_get_x (vec4);
    float y = graphene_vec4_get_y (vec4);
    float z = graphene_vec4_get_z (vec4);
    float w = graphene_vec4_get_w (vec4);
    gst_gl_shader_set_uniform_4f (shader, field_name, x, y, z, w);
  } else if (G_TYPE_CHECK_VALUE_TYPE ((value), GRAPHENE_TYPE_MATRIX)) {
    graphene_matrix_t *matrix = g_value_get_boxed (value);
    float matrix_f[16];
    graphene_matrix_to_float (matrix, matrix_f);
    gst_gl_shader_set_uniform_matrix_4fv (shader, field_name, 1, FALSE,
        matrix_f);
#endif
  } else {
    /* FIXME: Add support for unsigned ints, non 4x4 matrices, etc */
    GST_FIXME ("Don't know how to set the \'%s\' paramater.  Unknown type",
        field_name);
    return TRUE;
  }

  return TRUE;
}

static void
_update_uniforms (GstGLFilterShader * filtershader)
{
  if (filtershader->new_uniforms && filtershader->uniforms) {
    gst_gl_shader_use (filtershader->shader);

    gst_structure_foreach (filtershader->uniforms,
        (GstStructureForeachFunc) _set_uniform, filtershader->shader);
    filtershader->new_uniforms = FALSE;
  }
}

389 390 391 392 393 394 395 396 397
static GstGLShader *
_maybe_recompile_shader (GstGLFilterShader * filtershader)
{
  GstGLContext *context = GST_GL_BASE_FILTER (filtershader)->context;
  GstGLShader *shader;
  GError *error = NULL;

  GST_OBJECT_LOCK (filtershader);

398 399
  if (!filtershader->shader || filtershader->update_shader) {
    filtershader->update_shader = FALSE;
400 401
    GST_OBJECT_UNLOCK (filtershader);
    g_signal_emit (filtershader, gst_gl_shader_signals[SIGNAL_CREATE_SHADER], 0,
402 403 404 405 406 407
        &shader);
    GST_OBJECT_LOCK (filtershader);

    if (shader) {
      if (filtershader->shader)
        gst_object_unref (filtershader->shader);
408
      filtershader->new_source = FALSE;
409
      filtershader->shader = gst_object_ref (shader);
410 411
      filtershader->new_uniforms = TRUE;
      _update_uniforms (filtershader);
412 413 414 415 416
      GST_OBJECT_UNLOCK (filtershader);
      return shader;
    }
  }

417 418
  if (filtershader->shader) {
    shader = gst_object_ref (filtershader->shader);
419
    _update_uniforms (filtershader);
420 421 422 423
    GST_OBJECT_UNLOCK (filtershader);
    return shader;
  }

424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
  if (filtershader->new_source) {
    GstGLSLStage *stage;

    shader = gst_gl_shader_new (context);

    if (filtershader->vertex) {
      if (!(stage = gst_glsl_stage_new_with_string (context, GL_VERTEX_SHADER,
                  GST_GLSL_VERSION_NONE, GST_GLSL_PROFILE_NONE,
                  filtershader->vertex))) {
        g_set_error (&error, GST_GLSL_ERROR, GST_GLSL_ERROR_COMPILE,
            "Failed to create shader vertex stage");
        goto print_error;
      }
    } else {
      stage = gst_glsl_stage_new_default_vertex (context);
    }

    if (!gst_gl_shader_compile_attach_stage (shader, stage, &error)) {
      gst_object_unref (stage);
      goto print_error;
    }

    if (filtershader->fragment) {
      if (!(stage = gst_glsl_stage_new_with_string (context, GL_FRAGMENT_SHADER,
                  GST_GLSL_VERSION_NONE, GST_GLSL_PROFILE_NONE,
                  filtershader->fragment))) {
        g_set_error (&error, GST_GLSL_ERROR, GST_GLSL_ERROR_COMPILE,
            "Failed to create shader fragment stage");
        goto print_error;
      }
    } else {
      stage = gst_glsl_stage_new_default_fragment (context);
    }

    if (!gst_gl_shader_compile_attach_stage (shader, stage, &error)) {
      gst_object_unref (stage);
      goto print_error;
    }

    if (!gst_gl_shader_link (shader, &error)) {
      goto print_error;
    }
    if (filtershader->shader)
      gst_object_unref (filtershader->shader);
    filtershader->shader = gst_object_ref (shader);
    filtershader->new_source = FALSE;
470 471
    filtershader->new_uniforms = TRUE;
    _update_uniforms (filtershader);
472 473 474 475

    GST_OBJECT_UNLOCK (filtershader);
    return shader;
  } else if (filtershader->shader) {
476
    _update_uniforms (filtershader);
477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
    shader = gst_object_ref (filtershader->shader);
    GST_OBJECT_UNLOCK (filtershader);
    return shader;
  }

  return NULL;

print_error:
  if (shader) {
    gst_object_unref (shader);
    shader = NULL;
  }

  GST_OBJECT_UNLOCK (filtershader);
  GST_ELEMENT_ERROR (filtershader, RESOURCE, NOT_FOUND,
      ("%s", error->message), (NULL));
  return NULL;
}

496 497
static gboolean
gst_gl_filtershader_hcallback (GstGLFilter * filter, GstGLMemory * in_tex,
498 499 500
    gpointer stuff)
{
  GstGLFilterShader *filtershader = GST_GL_FILTERSHADER (filter);
501
  GstGLFuncs *gl = GST_GL_BASE_FILTER (filter)->context->gl_vtable;
502
  GstGLShader *shader;
503

504
  if (!(shader = _maybe_recompile_shader (filtershader)))
505
    return FALSE;
506

507 508
  gl->ClearColor (0.0, 0.0, 0.0, 1.0);
  gl->Clear (GL_COLOR_BUFFER_BIT);
509

510
  gst_gl_shader_use (shader);
511

512 513
  /* FIXME: propertise these */
  gst_gl_shader_set_uniform_1i (shader, "tex", 0);
514 515 516 517
  gst_gl_shader_set_uniform_1f (shader, "width",
      GST_VIDEO_INFO_WIDTH (&filter->out_info));
  gst_gl_shader_set_uniform_1f (shader, "height",
      GST_VIDEO_INFO_HEIGHT (&filter->out_info));
518 519 520
  gst_gl_shader_set_uniform_1f (shader, "time", filtershader->time);

  /* FIXME: propertise these */
Matthew Waters's avatar
Matthew Waters committed
521
  filter->draw_attr_position_loc =
522
      gst_gl_shader_get_attribute_location (shader, "a_position");
Matthew Waters's avatar
Matthew Waters committed
523
  filter->draw_attr_texture_loc =
524
      gst_gl_shader_get_attribute_location (shader, "a_texcoord");
525

526
  gl->ActiveTexture (GL_TEXTURE0);
527
  gl->BindTexture (GL_TEXTURE_2D, gst_gl_memory_get_texture_id (in_tex));
528

529
  gst_gl_filter_draw_fullscreen_quad (filter);
530 531

  gst_object_unref (shader);
532 533

  return TRUE;
534
}