Commit 088b4c0c authored by Sebastian Dröge's avatar Sebastian Dröge 🍵

overlaycomposition: New element that allows applications to draw...

overlaycomposition: New element that allows applications to draw GstVideoOverlayComposition on a stream

https://bugzilla.gnome.org/show_bug.cgi?id=797234
parent 596a4ee1
......@@ -501,6 +501,7 @@ AG_GST_CHECK_PLUGIN(audiotestsrc)
AG_GST_CHECK_PLUGIN(encoding)
AG_GST_CHECK_PLUGIN(videoconvert)
AG_GST_CHECK_PLUGIN(gio)
AG_GST_CHECK_PLUGIN(overlaycomposition)
AG_GST_CHECK_PLUGIN(playback)
AG_GST_CHECK_PLUGIN(audioresample)
AG_GST_CHECK_PLUGIN(rawparse)
......@@ -924,6 +925,7 @@ gst/audiotestsrc/Makefile
gst/encoding/Makefile
gst/videoconvert/Makefile
gst/gio/Makefile
gst/overlaycomposition/Makefile
gst/pbtypes/Makefile
gst/playback/Makefile
gst/rawparse/Makefile
......@@ -1027,6 +1029,7 @@ tests/examples/gl/gtk/filtervideooverlay/Makefile
tests/examples/gl/cocoa/Makefile
tests/examples/gl/sdl/Makefile
tests/examples/overlay/Makefile
tests/examples/overlaycomposition/Makefile
tests/examples/seek/Makefile
tests/examples/snapshot/Makefile
tests/examples/playback/Makefile
......
......@@ -84,6 +84,7 @@ EXTRA_HFILES = \
$(top_srcdir)/gst/playback/gstsubtitleoverlay.h \
$(top_srcdir)/gst/audiorate/gstaudiorate.h \
$(top_srcdir)/gst/audioresample/gstaudioresample.h \
$(top_srcdir)/gst/overlaycomposition/gstoverlaycomposition.h \
$(top_srcdir)/gst/rawparse/gstrawaudioparse.h \
$(top_srcdir)/gst/rawparse/gstrawvideoparse.h \
$(top_srcdir)/gst/rawparse/gstunalignedaudioparse.h \
......
......@@ -90,6 +90,7 @@
<xi:include href="xml/element-ogmvideoparse.xml" />
<xi:include href="xml/element-opusdec.xml" />
<xi:include href="xml/element-opusenc.xml" />
<xi:include href="xml/element-overlaycomposition.xml" />
<xi:include href="xml/element-parsebin.xml" />
<xi:include href="xml/element-playbin.xml" />
<xi:include href="xml/element-playbin3.xml" />
......@@ -146,6 +147,7 @@
<xi:include href="xml/plugin-ogg.xml" />
<xi:include href="xml/plugin-opus.xml" />
<xi:include href="xml/plugin-opengl.xml" />
<xi:include href="xml/plugin-overlaycomposition.xml" />
<xi:include href="xml/plugin-pango.xml" />
<xi:include href="xml/plugin-playback.xml" />
<xi:include href="xml/plugin-rawparse.xml" />
......
......@@ -1174,6 +1174,22 @@ GST_TYPE_OPUS_ENC
gst_opus_enc_get_type
</SECTION>
<SECTION>
<FILE>element-overlaycomposition</FILE>
<TITLE>overlaycomposition</TITLE>
GstOverlayComposition
<SUBSECTION Standard>
GstOverlayCompositionClass
GST_OVERLAY_COMPOSITION
GST_OVERLAY_COMPOSITION_CAST
GST_IS_OVERLAY_COMPOSITION
GST_OVERLAY_COMPOSITION_CLASS
GST_IS_OVERLAY_COMPOSITION_CLASS
GST_TYPE_OVERLAY_COMPOSITION
<SUBSECTION Private>
gst_overlay_composition_get_type
</SECTION>
<SECTION>
<FILE>element-parsebin</FILE>
<TITLE>parsebin</TITLE>
......
foreach plugin : ['adder', 'app', 'audioconvert', 'audiomixer', 'audiorate', 'audioresample',
'audiotestsrc', 'encoding', 'gio', 'pbtypes', 'playback', 'rawparse',
'subparse', 'tcp', 'typefind', 'videoconvert', 'videorate', 'videoscale',
'audiotestsrc', 'encoding', 'gio', 'overlaycomposition', 'pbtypes', 'playback',
'rawparse', 'subparse', 'tcp', 'typefind', 'videoconvert', 'videorate', 'videoscale',
'videotestsrc', 'volume']
if not get_option(plugin).disabled()
subdir(plugin)
......
noinst_HEADERS = gstoverlaycomposition.h
plugin_LTLIBRARIES = libgstoverlaycomposition.la
libgstoverlaycomposition_la_SOURCES = gstoverlaycomposition.c
libgstoverlaycomposition_la_CFLAGS = $(GST_PLUGINS_BASE_CFLAGS) $(GST_BASE_CFLAGS) $(GST_CFLAGS)
libgstoverlaycomposition_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS)
libgstoverlaycomposition_la_LIBADD = \
$(top_builddir)/gst-libs/gst/video/libgstvideo-$(GST_API_VERSION).la \
$(GST_BASE_LIBS) \
$(GST_LIBS)
This diff is collapsed.
/* GStreamer
* Copyright (C) 2018 Sebastian Dröge <sebastian@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include <gst/gst.h>
#include <gst/video/video.h>
#ifndef __GST_OVERLAY_COMPOSITION_H__
#define __GST_OVERLAY_COMPOSITION_H__
G_BEGIN_DECLS
#define GST_TYPE_OVERLAY_COMPOSITION \
(gst_overlay_composition_get_type())
#define GST_OVERLAY_COMPOSITION(obj) \
(G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_OVERLAY_COMPOSITION,GstOverlayComposition))
#define GST_OVERLAY_COMPOSITION_CLASS(klass) \
(G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_OVERLAY_COMPOSITION,GstOverlayCompositionClass))
#define GST_IS_OVERLAY_COMPOSITION(obj) \
(G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_OVERLAY_COMPOSITION))
#define GST_IS_OVERLAY_COMPOSITION_CLASS(klass) \
(G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_OVERLAY_COMPOSITION))
typedef struct _GstOverlayComposition GstOverlayComposition;
typedef struct _GstOverlayCompositionClass GstOverlayCompositionClass;
struct _GstOverlayComposition {
GstElement parent;
GstPad *sinkpad, *srcpad;
/* state */
GstSample *sample;
GstSegment segment;
GstCaps *caps;
GstVideoInfo info;
guint window_width, window_height;
gboolean attach_compo_to_buffer;
};
struct _GstOverlayCompositionClass {
GstElementClass parent_class;
};
GType gst_overlay_composition_get_type (void);
G_END_DECLS
#endif /* __GST_OVERLAY_COMPOSITION_H__ */
gstoverlaycomposition = library('gstoverlaycomposition',
'gstoverlaycomposition.c',
c_args : gst_plugins_base_args,
include_directories: [configinc, libsinc],
dependencies : [video_dep],
install : true,
install_dir : plugins_install_dir,
)
pkgconfig.generate(gstoverlaycomposition, install_dir : plugins_pkgconfig_install_dir)
......@@ -37,6 +37,7 @@ option('audioresample', type : 'feature', value : 'auto')
option('audiotestsrc', type : 'feature', value : 'auto')
option('encoding', type : 'feature', value : 'auto')
option('gio', type : 'feature', value : 'auto')
option('overlaycomposition', type : 'feature', value : 'auto')
option('pbtypes', type : 'feature', value : 'auto')
option('playback', type : 'feature', value : 'auto')
option('rawparse', type : 'feature', value : 'auto')
......
......@@ -220,6 +220,12 @@ else
check_audioresample =
endif
if USE_PLUGIN_OVERLAYCOMPOSITION
check_overlaycomposition = elements/overlaycomposition
else
check_overlaycomposition =
endif
if HAVE_CXX
cxx_checks = libs/gstlibscpp
else
......@@ -271,6 +277,7 @@ check_PROGRAMS = \
$(check_gl) \
$(check_ogg) \
$(check_opus) \
$(check_overlaycomposition) \
$(check_pango) \
$(check_playback) \
$(check_rawparse) \
......@@ -748,6 +755,16 @@ elements_volume_CFLAGS = \
$(GST_BASE_CFLAGS) \
$(AM_CFLAGS)
elements_overlaycomposition_CFLAGS = \
$(GST_PLUGINS_BASE_CFLAGS) \
$(GST_BASE_CFLAGS) \
$(AM_CFLAGS)
elements_overlaycomposition_LDADD = \
$(top_builddir)/gst-libs/gst/video/libgstvideo-@GST_API_VERSION@.la \
$(GST_BASE_LIBS) \
$(LDADD)
elements_vorbisdec_LDADD = \
$(LDADD) \
$(VORBIS_LIBS) \
......
......@@ -27,6 +27,7 @@ videoconvert
videoscale
videoscale-[1-6]
vorbistag
overlaycomposition
playbin
playbin-compressed
playbin-complex
......
/* GStreamer
*
* Copyright (C) 2018 Sebastian Dröge <sebastian@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <gst/gst.h>
#include <gst/check/gstcheck.h>
#include <gst/check/gstharness.h>
#include <gst/video/video.h>
#define VIDEO_WIDTH 320
#define VIDEO_HEIGHT 240
#define OVERLAY_WIDTH 16
#define OVERLAY_HEIGHT 16
#if G_BYTE_ORDER == G_LITTLE_ENDIAN
#define VIDEO_FORMAT_STR "BGRA"
#define VIDEO_FORMAT GST_VIDEO_FORMAT_BGRA
#else
#define VIDEO_FORMAT_STR "ARGB"
#define VIDEO_FORMAT GST_VIDEO_FORMAT_ARGB
#endif
#define VIDEO_CAPS "video/x-raw, " \
"format = (string) " VIDEO_FORMAT_STR ", " \
"width = (int) 320, " \
"height = (int) 240, " \
"framerate = (fraction) 30/1"
#define VIDEO_CAPS_WITH_META "video/x-raw(" GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION "), " \
"format = (string) " VIDEO_FORMAT_STR ", " \
"width = (int) 320, " \
"height = (int) 240, " \
"framerate = (fraction) 30/1"
static GstBuffer *
create_video_frame (void)
{
GstBuffer *buffer;
GstMapInfo map;
guint i;
buffer = gst_buffer_new_and_alloc (VIDEO_WIDTH * VIDEO_HEIGHT * 4);
gst_buffer_add_video_meta (buffer, GST_VIDEO_FRAME_FLAG_NONE, VIDEO_FORMAT,
VIDEO_WIDTH, VIDEO_HEIGHT);
gst_buffer_map (buffer, &map, GST_MAP_READWRITE);
for (i = 0; i < map.size; i += 4)
GST_WRITE_UINT32_LE (map.data + i, 0xff000000);
gst_buffer_unmap (buffer, &map);
return buffer;
}
static GstBuffer *
create_overlay_frame (guint32 color)
{
GstBuffer *buffer;
GstMapInfo map;
guint i;
buffer = gst_buffer_new_and_alloc (VIDEO_WIDTH * VIDEO_HEIGHT * 4);
gst_buffer_add_video_meta (buffer, GST_VIDEO_FRAME_FLAG_NONE, VIDEO_FORMAT,
VIDEO_WIDTH, VIDEO_HEIGHT);
gst_buffer_map (buffer, &map, GST_MAP_READWRITE);
for (i = 0; i < map.size; i += 4)
GST_WRITE_UINT32_LE (map.data + i, color);
gst_buffer_unmap (buffer, &map);
return buffer;
}
typedef struct
{
gboolean valid;
GstVideoInfo info;
guint expected_window_width, expected_window_height;
GstVideoOverlayComposition *comp;
} State;
static void
on_caps_changed (GstElement * element, GstCaps * caps, guint window_width,
guint window_height, State * s)
{
fail_unless (gst_video_info_from_caps (&s->info, caps));
s->valid = TRUE;
fail_unless_equals_int (s->expected_window_width, window_width);
fail_unless_equals_int (s->expected_window_height, window_height);
}
static GstVideoOverlayComposition *
on_draw (GstElement * element, GstSample * sample, State * s)
{
fail_unless (s->valid);
fail_unless (GST_IS_SAMPLE (sample));
return gst_video_overlay_composition_ref (s->comp);
}
GST_START_TEST (render_fallback)
{
GstHarness *h;
GstVideoOverlayComposition *comp;
GstVideoOverlayRectangle *rect;
GstBuffer *buffer, *overlay;
State s = { 0, };
GstMapInfo map;
guint x, y;
h = gst_harness_new ("overlaycomposition");
g_signal_connect (h->element, "draw", G_CALLBACK (on_draw), &s);
g_signal_connect (h->element, "caps-changed", G_CALLBACK (on_caps_changed),
&s);
buffer = create_video_frame ();
overlay = create_overlay_frame (0x80ffffff);
rect =
gst_video_overlay_rectangle_new_raw (overlay, 32, 32, OVERLAY_WIDTH,
OVERLAY_HEIGHT, GST_VIDEO_OVERLAY_FORMAT_FLAG_NONE);
gst_buffer_unref (overlay);
comp = gst_video_overlay_composition_new (rect);
gst_video_overlay_rectangle_unref (rect);
s.comp = comp;
s.expected_window_width = VIDEO_WIDTH;
s.expected_window_height = VIDEO_HEIGHT;
gst_harness_set_src_caps_str (h, VIDEO_CAPS);
buffer = gst_harness_push_and_pull (h, buffer);
gst_buffer_map (buffer, &map, GST_MAP_READ);
fail_unless_equals_int (map.size, VIDEO_WIDTH * VIDEO_HEIGHT * 4);
for (y = 0; y < VIDEO_HEIGHT; y++) {
for (x = 0; x < VIDEO_WIDTH; x++) {
guint32 val = GST_READ_UINT32_LE (map.data + y * VIDEO_WIDTH * 4 + x * 4);
guint32 expected_val;
if ((x >= 32 && x < 48) && (y >= 32 && y < 48)) {
expected_val = 0xff808080;
} else {
expected_val = 0xff000000;
}
fail_unless (val == expected_val, "Expected %08x but got %08x at (%u,%u)",
expected_val, val, x, y);
}
}
gst_buffer_unmap (buffer, &map);
gst_buffer_unref (buffer);
gst_video_overlay_composition_unref (s.comp);
gst_harness_teardown (h);
}
GST_END_TEST;
GST_START_TEST (render_fallback_2)
{
GstHarness *h;
GstVideoOverlayComposition *comp;
GstVideoOverlayRectangle *rect;
GstBuffer *buffer, *overlay;
State s = { 0, };
GstMapInfo map;
guint x, y;
h = gst_harness_new ("overlaycomposition");
g_signal_connect (h->element, "draw", G_CALLBACK (on_draw), &s);
g_signal_connect (h->element, "caps-changed", G_CALLBACK (on_caps_changed),
&s);
overlay = create_overlay_frame (0xffff0000);
rect =
gst_video_overlay_rectangle_new_raw (overlay, 32, 32, OVERLAY_WIDTH,
OVERLAY_HEIGHT, GST_VIDEO_OVERLAY_FORMAT_FLAG_NONE);
gst_buffer_unref (overlay);
comp = gst_video_overlay_composition_new (rect);
gst_video_overlay_rectangle_unref (rect);
overlay = create_overlay_frame (0xff0000ff);
rect =
gst_video_overlay_rectangle_new_raw (overlay, 64, 64, OVERLAY_WIDTH,
OVERLAY_HEIGHT, GST_VIDEO_OVERLAY_FORMAT_FLAG_NONE);
gst_buffer_unref (overlay);
gst_video_overlay_composition_add_rectangle (comp, rect);
gst_video_overlay_rectangle_unref (rect);
s.comp = comp;
s.expected_window_width = VIDEO_WIDTH;
s.expected_window_height = VIDEO_HEIGHT;
gst_harness_set_src_caps_str (h, VIDEO_CAPS);
buffer = create_video_frame ();
buffer = gst_harness_push_and_pull (h, buffer);
gst_buffer_map (buffer, &map, GST_MAP_READ);
fail_unless_equals_int (map.size, VIDEO_WIDTH * VIDEO_HEIGHT * 4);
for (y = 0; y < VIDEO_HEIGHT; y++) {
for (x = 0; x < VIDEO_WIDTH; x++) {
guint32 val = GST_READ_UINT32_LE (map.data + y * VIDEO_WIDTH * 4 + x * 4);
guint32 expected_val;
if ((x >= 32 && x < 48) && (y >= 32 && y < 48)) {
expected_val = 0xffff0000;
} else if ((x >= 64 && x < 80) && (y >= 64 && y < 80)) {
expected_val = 0xff0000ff;
} else {
expected_val = 0xff000000;
}
fail_unless (val == expected_val, "Expected %08x but got %08x at (%u,%u)",
expected_val, val, x, y);
}
}
gst_buffer_unmap (buffer, &map);
gst_buffer_unref (buffer);
gst_video_overlay_composition_unref (s.comp);
gst_harness_teardown (h);
}
GST_END_TEST;
GST_START_TEST (render_meta)
{
GstHarness *h;
GstVideoOverlayComposition *comp;
GstVideoOverlayRectangle *rect;
GstBuffer *buffer, *overlay;
State s = { 0, };
GstMapInfo map;
guint x, y;
GstVideoOverlayCompositionMeta *meta;
h = gst_harness_new ("overlaycomposition");
g_signal_connect (h->element, "draw", G_CALLBACK (on_draw), &s);
g_signal_connect (h->element, "caps-changed", G_CALLBACK (on_caps_changed),
&s);
overlay = create_overlay_frame (0xffff0000);
rect =
gst_video_overlay_rectangle_new_raw (overlay, 32, 32, OVERLAY_WIDTH,
OVERLAY_HEIGHT, GST_VIDEO_OVERLAY_FORMAT_FLAG_NONE);
gst_buffer_unref (overlay);
comp = gst_video_overlay_composition_new (rect);
gst_video_overlay_rectangle_unref (rect);
overlay = create_overlay_frame (0xff0000ff);
rect =
gst_video_overlay_rectangle_new_raw (overlay, 64, 64, OVERLAY_WIDTH,
OVERLAY_HEIGHT, GST_VIDEO_OVERLAY_FORMAT_FLAG_NONE);
gst_buffer_unref (overlay);
gst_video_overlay_composition_add_rectangle (comp, rect);
gst_video_overlay_rectangle_unref (rect);
s.comp = comp;
s.expected_window_width = VIDEO_WIDTH;
s.expected_window_height = VIDEO_HEIGHT;
gst_harness_add_propose_allocation_meta (h,
GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL);
gst_harness_set_src_caps_str (h, VIDEO_CAPS);
buffer = create_video_frame ();
buffer = gst_harness_push_and_pull (h, buffer);
gst_buffer_map (buffer, &map, GST_MAP_READ);
fail_unless_equals_int (map.size, VIDEO_WIDTH * VIDEO_HEIGHT * 4);
for (y = 0; y < VIDEO_HEIGHT; y++) {
for (x = 0; x < VIDEO_WIDTH; x++) {
guint32 val = GST_READ_UINT32_LE (map.data + y * VIDEO_WIDTH * 4 + x * 4);
guint32 expected_val = 0xff000000;
fail_unless (val == expected_val, "Expected %08x but got %08x at (%u,%u)",
expected_val, val, x, y);
}
}
gst_buffer_unmap (buffer, &map);
meta = gst_buffer_get_video_overlay_composition_meta (buffer);
fail_unless (meta);
fail_unless (meta->overlay == s.comp);
gst_buffer_unref (buffer);
gst_video_overlay_composition_unref (s.comp);
gst_harness_teardown (h);
}
GST_END_TEST;
static Suite *
overlaycomposition_suite (void)
{
Suite *s = suite_create ("overlaycomposition");
TCase *tc = tcase_create ("general");
suite_add_tcase (s, tc);
tcase_add_test (tc, render_fallback);
tcase_add_test (tc, render_fallback_2);
tcase_add_test (tc, render_meta);
return s;
}
GST_CHECK_MAIN (overlaycomposition);
......@@ -43,6 +43,7 @@ base_tests = [
[ 'elements/multifdsink.c', not core_conf.has('HAVE_SYS_SOCKET_H') or not core_conf.has('HAVE_UNISTD_H') ],
# FIXME: multisocketsink test on windows/msvc
[ 'elements/multisocketsink.c', not core_conf.has('HAVE_SYS_SOCKET_H') or not core_conf.has('HAVE_UNISTD_H') ],
[ 'elements/overlaycomposition.c' ],
[ 'elements/playbin.c' ],
[ 'elements/playbin-complex.c', not ogg_dep.found() ],
[ 'elements/playsink.c' ],
......
......@@ -8,8 +8,8 @@ else
GL_DIR=
endif
SUBDIRS = app audio decodebin_next dynamic fft gio $(GL_DIR) $(GTK_SUBDIRS) overlay playrec encoding
DIST_SUBDIRS = app audio dynamic decodebin_next fft gio gl playback overlay seek snapshot playrec encoding
SUBDIRS = app audio decodebin_next dynamic fft gio $(GL_DIR) $(GTK_SUBDIRS) overlay overlaycomposition playrec encoding
DIST_SUBDIRS = app audio dynamic decodebin_next fft gio gl playback overlay overlaycomposition seek snapshot playrec encoding
include $(top_srcdir)/common/parallel-subdirs.mak
......@@ -9,6 +9,7 @@ if build_gstgl
subdir('gl')
endif
subdir('overlay')
subdir('overlaycomposition')
subdir('playback')
subdir('playrec')
subdir('seek')
......
noinst_PROGRAMS = overlaycomposition
overlaycomposition_SOURCES = overlaycomposition.c
overlaycomposition_CFLAGS = $(GST_PLUGINS_BASE_CFLAGS) $(GST_CFLAGS) $(GIO_CFLAGS)
overlaycomposition_LDADD = $(GST_PLUGINS_BASE_LIBS) -lgstvideo-$(GST_API_VERSION) $(GST_LIBS) $(GIO_LIBS) $(LIBM)
executable('overlaycomposition', 'overlaycomposition.c',
dependencies: [video_dep, gst_dep, gio_dep, libm],
c_args : gst_plugins_good_args,
include_directories : [configinc],
install: false)
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment