WIP: livevideorate, liveaudiorate: Add a new videorate and audiorate elements based on aggregator

New videorate implementation based on videoaggregator. Unlike existing videorate element, this new element will produce output frames even if there is no incoming buffers within a configured timeout value. The timeout value can be configured via "min-upstream-latency". Note that actual timeout value will be re-calculated by using upstream latency value.

Not only duplicating previous frame when input buffer is too late, users can select fill mode via "fill-mode" property whether output frame should be duplication of previous image or a new black image.

gst/liveaudiorate/gstliveaudiorate.c

+/* GStreamer
+ * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstliveaudiorate.h"
+#include <string.h>
+
+GST_DEBUG_CATEGORY_STATIC (gst_live_audio_rate_debug);
+#define GST_CAT_DEFAULT gst_live_audio_rate_debug
+
+static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
+    GST_PAD_SRC,
+    GST_PAD_ALWAYS,
+    GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE (GST_AUDIO_FORMATS_ALL)
+        ", layout = (string) interleaved")
+    );
+
+static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
+    GST_PAD_SINK,
+    GST_PAD_ALWAYS,
+    GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE (GST_AUDIO_FORMATS_ALL)
+        ", layout = (string) interleaved")
+    );
+
+struct _GstLiveAudioRate
+{
+  GstAudioAggregator parent;
+  GstAudioAggregatorPad *sinkpad;
+};
+
+static gboolean gst_live_audio_rate_sink_event (GstAggregator * agg,
+    GstAggregatorPad * agg_pad, GstEvent * event);
+static gboolean
+gst_live_audio_rate_aggregate_one_buffer (GstAudioAggregator * aagg,
+    GstAudioAggregatorPad * aaggpad, GstBuffer * inbuf, guint in_offset,
+    GstBuffer * outbuf, guint out_offset, guint num_frames);
+
+#define gst_live_audio_rate_parent_class parent_class
+G_DEFINE_TYPE (GstLiveAudioRate, gst_live_audio_rate,
+    GST_TYPE_AUDIO_AGGREGATOR);
+
+static void
+gst_live_audio_rate_class_init (GstLiveAudioRateClass * klass)
+{
+  GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
+  GstAggregatorClass *agg_class = GST_AGGREGATOR_CLASS (klass);
+  GstAudioAggregatorClass *aagg_class = GST_AUDIO_AGGREGATOR_CLASS (klass);
+
+  /* we have 1:1 static pads */
+  element_class->request_new_pad = NULL;
+  element_class->release_pad = NULL;
+
+  gst_element_class_set_static_metadata (element_class,
+      "Live Audio Rate",
+      "Filter/Effect/Audio",
+      "Inserts silence or drops audio samples to make a perfectly timestamped stream",
+      "Seungha Yang <seungha@centricular.com>");
+
+  gst_element_class_add_static_pad_template_with_gtype (element_class,
+      &sink_template, GST_TYPE_AUDIO_AGGREGATOR_PAD);
+  gst_element_class_add_static_pad_template_with_gtype (element_class,
+      &src_template, GST_TYPE_AUDIO_AGGREGATOR_PAD);
+
+  agg_class->sink_event = GST_DEBUG_FUNCPTR (gst_live_audio_rate_sink_event);
+  aagg_class->aggregate_one_buffer =
+      GST_DEBUG_FUNCPTR (gst_live_audio_rate_aggregate_one_buffer);
+
+  GST_DEBUG_CATEGORY_INIT (gst_live_audio_rate_debug, "liveaudiorate", 0,
+      "LiveAudioRate stream fixer");
+}
+
+static void
+gst_live_audio_rate_init (GstLiveAudioRate * self)
+{
+  GstPadTemplate *templ;
+
+  templ = gst_static_pad_template_get (&sink_template);
+  self->sinkpad = g_object_new (GST_TYPE_AUDIO_AGGREGATOR_PAD,
+      "name", "sink", "direction", GST_PAD_SINK, "template", templ, NULL);
+  gst_object_unref (templ);
+  gst_element_add_pad (GST_ELEMENT_CAST (self), GST_PAD_CAST (self->sinkpad));
+
+  /* we are live element */
+  gst_aggregator_set_live (GST_AGGREGATOR_CAST (self), TRUE);
+}
+
+static gboolean
+gst_live_audio_rate_sink_event (GstAggregator * agg, GstAggregatorPad * agg_pad,
+    GstEvent * event)
+{
+  switch (GST_EVENT_TYPE (event)) {
+    case GST_EVENT_SEGMENT:{
+      const GstSegment *segment;
+
+      /* Only single input, so we can use upstream segment without
+       * update */
+      gst_event_parse_segment (event, &segment);
+      gst_aggregator_update_segment (agg, segment);
+      break;
+    }
+    default:
+      break;
+  }
+
+  return GST_AGGREGATOR_CLASS (parent_class)->sink_event (agg, agg_pad, event);
+}
+
+static gboolean
+gst_live_audio_rate_aggregate_one_buffer (GstAudioAggregator * aagg,
+    GstAudioAggregatorPad * aaggpad, GstBuffer * inbuf, guint in_offset,
+    GstBuffer * outbuf, guint out_offset, guint num_frames)
+{
+  GstAggregator *agg = GST_AGGREGATOR (aagg);
+  GstAudioAggregatorPad *srcpad = GST_AUDIO_AGGREGATOR_PAD (agg->srcpad);
+  GstMapInfo in_map;
+  GstMapInfo out_map;
+  guint8 *in_data, *out_data;
+  gint bpf;
+
+  gst_buffer_map (inbuf, &in_map, GST_MAP_READ);
+  gst_buffer_map (outbuf, &out_map, GST_MAP_READWRITE);
+
+  bpf = GST_AUDIO_INFO_BPF (&srcpad->info);
+  in_data = (guint8 *) in_map.data + in_offset * bpf;
+  out_data = (guint8 *) out_map.data + out_offset * bpf;
+
+  memcpy (out_data, in_data, num_frames * bpf);
+
+  gst_buffer_unmap (outbuf, &out_map);
+  gst_buffer_unmap (inbuf, &in_map);
+
+  return TRUE;
+}
+
+static gboolean
+plugin_init (GstPlugin * plugin)
+{
+  return gst_element_register (plugin, "liveaudiorate", GST_RANK_NONE,
+      GST_TYPE_LIVE_AUDIO_RATE);
+}
+
+GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
+    GST_VERSION_MINOR,
+    liveaudiorate,
+    "Adjusts audio frames",
+    plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)