commit 940b30d3442446e9ffd6ef94e77ccc004c9cbca8 Author: Dirk-Jan C. Binnema Date: Tue Jan 23 16:31:55 2018 rtph264pay: Support STAP-A bundling Add a new property "do-aggregate"[1] to the H.264 RTP payloader, which, when set, enables STAP-A aggregation, as per RFC-6184[2]. With aggregation enabled, SPS/PPS packets are bundled instead of sent immediately, up until the MTU size. An attempt is made to change the existing payloader as little as possible. [1] the property-name is kept generic since it might applicable more widely, e.g. STAP-B, MTAP [2] https://tools.ietf.org/html/rfc6184#section-5.7 diff --git a/gst/rtp/gstrtph264pay.c b/gst/rtp/gstrtph264pay.c index 73f080a7b..5578c0627 100644 --- a/gst/rtp/gstrtph264pay.c +++ b/gst/rtp/gstrtph264pay.c @@ -35,9 +35,10 @@ #include "gstrtputils.h" -#define IDR_TYPE_ID 5 -#define SPS_TYPE_ID 7 -#define PPS_TYPE_ID 8 +#define IDR_TYPE_ID 5 +#define SPS_TYPE_ID 7 +#define PPS_TYPE_ID 8 +#define STAP_A_TYPE_ID 24 GST_DEBUG_CATEGORY_STATIC (rtph264pay_debug); #define GST_CAT_DEFAULT (rtph264pay_debug) @@ -69,12 +70,14 @@ GST_STATIC_PAD_TEMPLATE ("src", #define DEFAULT_SPROP_PARAMETER_SETS NULL #define DEFAULT_CONFIG_INTERVAL 0 +#define DEFAULT_DO_AGGREGATE FALSE enum { PROP_0, PROP_SPROP_PARAMETER_SETS, - PROP_CONFIG_INTERVAL + PROP_CONFIG_INTERVAL, + PROP_DO_AGGREGATE, }; #define IS_ACCESS_UNIT(x) (((x) > 0x00) && ((x) < 0x06)) @@ -97,6 +100,8 @@ static gboolean gst_rtp_h264_pay_sink_event (GstRTPBasePayload * payload, static GstStateChangeReturn gst_rtp_h264_pay_change_state (GstElement * element, GstStateChange transition); +static void gst_rtp_h264_reset_bundle (GstRtpH264Pay * rtph264pay); + #define gst_rtp_h264_pay_parent_class parent_class G_DEFINE_TYPE (GstRtpH264Pay, gst_rtp_h264_pay, GST_TYPE_RTP_BASE_PAYLOAD); @@ -133,6 +138,16 @@ gst_rtp_h264_pay_class_init (GstRtpH264PayClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) ); + + g_object_class_install_property (G_OBJECT_CLASS (klass), + PROP_DO_AGGREGATE, + g_param_spec_boolean ("do-aggregate", + "Attempt to use aggregate packets", + "Bundle suitable SPS/PPS NAL units into STAP-A " + "aggregate packets. ", + FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) + ); + gobject_class->finalize = gst_rtp_h264_pay_finalize; gst_element_class_add_static_pad_template (gstelement_class, @@ -168,10 +183,13 @@ gst_rtp_h264_pay_init (GstRtpH264Pay * rtph264pay) (GDestroyNotify) gst_buffer_unref); rtph264pay->last_spspps = -1; rtph264pay->spspps_interval = DEFAULT_CONFIG_INTERVAL; + rtph264pay->do_aggregate = DEFAULT_DO_AGGREGATE; rtph264pay->delta_unit = FALSE; rtph264pay->discont = FALSE; rtph264pay->adapter = gst_adapter_new (); + + gst_rtp_h264_reset_bundle (rtph264pay); } static void @@ -196,6 +214,7 @@ gst_rtp_h264_pay_finalize (GObject * object) g_free (rtph264pay->sprop_parameter_sets); g_object_unref (rtph264pay->adapter); + gst_rtp_h264_reset_bundle (rtph264pay); G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -727,10 +746,37 @@ gst_rtp_h264_pay_decode_nal (GstRtpH264Pay * payloader, } static GstFlowReturn -gst_rtp_h264_pay_payload_nal (GstRTPBasePayload * basepayload, +gst_rtp_h264_pay_payload_nal_single (GstRTPBasePayload * basepayload, + GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean end_of_au, + gboolean delta_unit, gboolean discont); + +static GstFlowReturn gst_rtp_h264_send_bundle (GstRTPBasePayload * basepayload); + +static GstFlowReturn +gst_rtp_h264_pay_payload_nal_bundle (GstRTPBasePayload * basepayload, GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean end_of_au, gboolean delta_unit, gboolean discont); + +static GstFlowReturn +gst_rtp_h264_pay_payload_nal (GstRTPBasePayload * basepayload, + GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean end_of_au, + gboolean delta_unit, gboolean discont) +{ + GstRtpH264Pay *rtph264pay; + + rtph264pay = GST_RTP_H264_PAY (basepayload); + + if (rtph264pay->do_aggregate) + return gst_rtp_h264_pay_payload_nal_bundle (basepayload, paybuf, dts, pts, + end_of_au, delta_unit, discont); + else + return gst_rtp_h264_pay_payload_nal_single (basepayload, paybuf, dts, pts, + end_of_au, delta_unit, discont); +} + + + static GstFlowReturn gst_rtp_h264_pay_send_sps_pps (GstRTPBasePayload * basepayload, GstRtpH264Pay * rtph264pay, GstClockTime dts, GstClockTime pts) @@ -768,6 +814,8 @@ gst_rtp_h264_pay_send_sps_pps (GstRTPBasePayload * basepayload, } } + gst_rtp_h264_send_bundle (basepayload); /* don't wait, send now */ + if (pts != -1 && sent_all_sps_pps) rtph264pay->last_spspps = pts; @@ -780,7 +828,7 @@ gst_rtp_h264_pay_send_sps_pps (GstRTPBasePayload * basepayload, * GST_BUFFER_FLAG_DISCONT flag. */ static GstFlowReturn -gst_rtp_h264_pay_payload_nal (GstRTPBasePayload * basepayload, +gst_rtp_h264_pay_payload_nal_single (GstRTPBasePayload * basepayload, GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean end_of_au, gboolean delta_unit, gboolean discont) { @@ -851,6 +899,7 @@ gst_rtp_h264_pay_payload_nal (GstRTPBasePayload * basepayload, /* we need to send SPS/PPS now first. FIXME, don't use the pts for * checking when we need to send SPS/PPS but convert to running_time first. */ rtph264pay->send_spspps = FALSE; + gst_rtp_h264_send_bundle (basepayload); /* clear up */ ret = gst_rtp_h264_pay_send_sps_pps (basepayload, rtph264pay, dts, pts); if (ret != GST_FLOW_OK) { gst_buffer_unref (paybuf); @@ -987,6 +1036,184 @@ gst_rtp_h264_pay_payload_nal (GstRTPBasePayload * basepayload, return ret; } + +static void +gst_rtp_h264_reset_bundle (GstRtpH264Pay * rtph264pay) +{ + g_clear_pointer (&rtph264pay->bundle, gst_buffer_unref); + rtph264pay->bundle_nri = 0; + rtph264pay->bundle_f_bit = 0; + rtph264pay->bundle_n = 0; +} + + +static GstFlowReturn +gst_rtp_h264_send_bundle (GstRTPBasePayload * basepayload) +{ + guint8 hdr; + GstFlowReturn ret; + GstRtpH264Pay *rtph264pay; + + rtph264pay = GST_RTP_H264_PAY (basepayload); + + if (!rtph264pay->bundle) { + GST_DEBUG_OBJECT (rtph264pay, "no bundle, nothing to do"); + return GST_FLOW_OK; + } + + hdr = (((rtph264pay->bundle_f_bit << 7) & 0x80) | + ((rtph264pay->bundle_nri << 5) & 0x60) | + (STAP_A_TYPE_ID & 0x1f)); /*24 */ ; + if (!gst_buffer_is_writable (rtph264pay->bundle)) + rtph264pay->bundle = gst_buffer_make_writable (rtph264pay->bundle); + + gst_buffer_fill (rtph264pay->bundle, 0, &hdr, sizeof (hdr)); + + GST_DEBUG_OBJECT (rtph264pay, + "sending STAP-A bundle (n=%u) [0x%0x] %" GST_PTR_FORMAT, + rtph264pay->bundle_n, hdr, rtph264pay->bundle); + +#if 0 /* debugging only */ + { + guint8 *dest; + guint u, len = gst_buffer_get_size (rtph264pay->bundle); + dest = g_new (guint8, len); + gst_buffer_extract (rtph264pay->bundle, 0, dest, len); + + for (u = 0; u != len; ++u) { + g_printerr ("%02x ", dest[u]); + if ((u + 1) % 16 == 0) + g_printerr ("\n"); + } + g_free (dest); + g_printerr ("\n"); + } +#endif /* 0 */ + + ret = gst_rtp_h264_pay_payload_nal_single (basepayload, + gst_buffer_ref (rtph264pay->bundle), + GST_BUFFER_DTS (rtph264pay->bundle), + GST_BUFFER_PTS (rtph264pay->bundle), TRUE, FALSE, FALSE); + + gst_rtp_h264_reset_bundle (rtph264pay); + + return ret; +} + + +/* either bundle the buffer, or send it immediately */ +static gboolean +gst_rtp_h264_pay_payload_nal_bundle (GstRTPBasePayload * basepayload, + GstBuffer * paybuf, + GstClockTime dts, GstClockTime pts, + gboolean end_of_au, gboolean delta_unit, gboolean discont) +{ + GstFlowReturn ret; + guint pay_size, bundle_size; + guint mtu; + GstRtpH264Pay *rtph264pay; + gboolean did_bundle; + guint16 nalu_len; + GstBuffer *len_buf; + guint8 hdr, nal_type; + + did_bundle = FALSE; + rtph264pay = GST_RTP_H264_PAY (basepayload); + + gst_buffer_extract (paybuf, 0, &hdr, 1); + nal_type = hdr & 0x1f; + + /* only bundle SPS / PPS; anything else won't fit */ + if (nal_type != 7 && nal_type != 8) + goto actually_send; + + if (rtph264pay->bundle) + bundle_size = gst_buffer_get_size (rtph264pay->bundle); + else + bundle_size = 0; + + mtu = GST_RTP_BASE_PAYLOAD_MTU (rtph264pay); + pay_size = gst_buffer_get_size (paybuf); + + if (bundle_size + pay_size + 3 > mtu) { + GST_DEBUG_OBJECT (rtph264pay, "packet does not fit in bundle"); + goto actually_send; + } + + if (rtph264pay->bundle) { + if (GST_BUFFER_PTS (rtph264pay->bundle) != pts || + GST_BUFFER_DTS (rtph264pay->bundle) != dts) { + GST_DEBUG_OBJECT (rtph264pay, "timestamps don't match"); + goto actually_send; + } + } else { + rtph264pay->bundle = gst_buffer_new_allocate (NULL, 1, NULL); + GST_BUFFER_DTS (rtph264pay->bundle) = dts; + GST_BUFFER_PTS (rtph264pay->bundle) = pts; + GST_DEBUG_OBJECT (rtph264pay, + "creating STAP-A aggregate with %" GST_PTR_FORMAT, paybuf); + } + + /* when we get here we did (attempt to) bundle the packet + * ie, we should not try to send it as a single packet. */ + did_bundle = TRUE; + + nalu_len = GUINT16_TO_BE (pay_size); + len_buf = gst_buffer_new_allocate (NULL, 2, NULL); + gst_buffer_fill (len_buf, 0, &nalu_len, sizeof (nalu_len)); + gst_buffer_copy_into (rtph264pay->bundle, len_buf, + GST_BUFFER_COPY_MEMORY, 0, -1); + GST_DEBUG_OBJECT (rtph264pay, "adding part to bundle [%04x] (%u)", + nalu_len, (guint) gst_buffer_get_size (len_buf)); + gst_buffer_unref (len_buf); + + gst_buffer_extract (paybuf, 0, &hdr, 1); + GST_DEBUG_OBJECT (rtph264pay, "bundling nal type %u 0x%02x", hdr & 0x1f, hdr); + rtph264pay->bundle_nri = MAX (rtph264pay->bundle_nri, (hdr & 0x60) >> 5); + rtph264pay->bundle_f_bit = MAX (rtph264pay->bundle_f_bit, (hdr & 0x80) >> 7); + + if (!gst_buffer_copy_into (rtph264pay->bundle, paybuf, + GST_BUFFER_COPY_MEMORY, 0, -1)) { + GST_DEBUG_OBJECT (rtph264pay, "failed to add buffer to aggregate"); + goto actually_send; + } else { + GST_INFO_OBJECT (rtph264pay, "bundled %u bytes into STAP-A (n=%u)", + pay_size, ++rtph264pay->bundle_n); + g_clear_pointer (&paybuf, gst_buffer_unref); + return GST_FLOW_OK; + } + +actually_send: + + /* if we have a bundle, send that first */ + if (rtph264pay->bundle) { + /* if the next packet (after this SPS/PPS bundle) is an IDR, fixup + * the timestamps.*/ + if (nal_type == IDR_TYPE_ID) { + GST_BUFFER_DTS (rtph264pay->bundle) = dts; + GST_BUFFER_PTS (rtph264pay->bundle) = pts; + } + + ret = gst_rtp_h264_send_bundle (basepayload); + if (ret != GST_FLOW_OK) { + GST_WARNING_OBJECT (rtph264pay, "error sending bundle"); + return ret; + } + } + + /* if we bundled the data, we're done now. */ + if (did_bundle) { + g_clear_pointer (&paybuf, gst_buffer_unref); + return ret; + } + + /* could not bundle this packet; so send it */ + /* GST_DEBUG_OBJECT(rtph264pay, "send single %u %u", nal_type, pay_size); */ + return gst_rtp_h264_pay_payload_nal_single (basepayload, paybuf, dts, pts, + end_of_au, delta_unit, discont); +} + + static GstFlowReturn gst_rtp_h264_pay_handle_buffer (GstRTPBasePayload * basepayload, GstBuffer * buffer) @@ -1389,6 +1616,9 @@ gst_rtp_h264_pay_set_property (GObject * object, guint prop_id, case PROP_CONFIG_INTERVAL: rtph264pay->spspps_interval = g_value_get_int (value); break; + case PROP_DO_AGGREGATE: + rtph264pay->do_aggregate = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1410,6 +1640,9 @@ gst_rtp_h264_pay_get_property (GObject * object, guint prop_id, case PROP_CONFIG_INTERVAL: g_value_set_int (value, rtph264pay->spspps_interval); break; + case PROP_DO_AGGREGATE: + g_value_set_boolean (value, rtph264pay->do_aggregate); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; diff --git a/gst/rtp/gstrtph264pay.h b/gst/rtp/gstrtph264pay.h index c5a5e9fb6..bc64ad285 100644 --- a/gst/rtp/gstrtph264pay.h +++ b/gst/rtp/gstrtph264pay.h @@ -79,6 +79,13 @@ struct _GstRtpH264Pay gboolean delta_unit; /* TRUE if the next NALU processed should have the DISCONT flag */ gboolean discont; + + /* aggregate buffers*/ + GstBuffer *bundle; + guint8 bundle_f_bit; + guint8 bundle_nri; + guint bundle_n; + gboolean do_aggregate; }; struct _GstRtpH264PayClass