Commit ae644937 authored by Mark Nauwelaerts's avatar Mark Nauwelaerts Committed by Tim-Philipp Müller
Browse files

qtmux: adjust nasty case timestamp tracking

That is, all sorts of problems arise with re-ordered input timestamps that
tend to defy automagic handling for every case, so allow for a few variations
that can be tried depending on circumstances.

Also try to document accordingly.

Also fixes #638288.
parent 62c72846
......@@ -82,6 +82,18 @@
* <link linkend="GstQTMux--streamable">streamable</link> allows foregoing to add
* index metadata (at the end of file).
*
* <link linkend="GstQTMux--dts-method">dts-method</link> allows selecting a
* method for managing input timestamps (stay tuned for 0.11 to have this
* automagically settled). The default delta/duration method should handle nice
* (aka perfect streams) just fine, but may experience problems otherwise
* (e.g. input stream with re-ordered B-frames and/or with frame dropping).
* The re-ordering approach re-assigns incoming timestamps in ascending order
* to incoming buffers and offers an alternative in such cases. In cases where
* that might fail, the remaining method can be tried, which is exact and
* according to specs, but might experience playback on not so spec-wise players.
* Note that this latter approach also requires one to enable
* <link linkend="GstQTMux--presentation-timestamp">presentation-timestamp</link>.
*
* <refsect2>
* <title>Example pipelines</title>
* |[
......@@ -128,6 +140,36 @@
GST_DEBUG_CATEGORY_STATIC (gst_qt_mux_debug);
#define GST_CAT_DEFAULT gst_qt_mux_debug
enum
{
DTS_METHOD_DD,
DTS_METHOD_REORDER,
DTS_METHOD_ASC
};
static GType
gst_qt_mux_dts_method_get_type (void)
{
static GType gst_qt_mux_dts_method = 0;
if (!gst_qt_mux_dts_method) {
static const GEnumValue dts_methods[] = {
{DTS_METHOD_DD, "delta/duration", "dd"},
{DTS_METHOD_REORDER, "reorder", "reorder"},
{DTS_METHOD_ASC, "ascending", "asc"},
{0, NULL, NULL},
};
gst_qt_mux_dts_method =
g_enum_register_static ("GstQTMuxDtsMethods", dts_methods);
}
return gst_qt_mux_dts_method;
}
#define GST_TYPE_QT_MUX_DTS_METHOD \
(gst_qt_mux_dts_method_get_type ())
/* QTMux signals and args */
enum
{
......@@ -140,12 +182,13 @@ enum
PROP_0,
PROP_MOVIE_TIMESCALE,
PROP_TRAK_TIMESCALE,
PROP_DO_CTTS,
PROP_FAST_START,
PROP_FAST_START_TEMP_FILE,
PROP_MOOV_RECOV_FILE,
PROP_FRAGMENT_DURATION,
PROP_STREAMABLE
PROP_STREAMABLE,
PROP_DTS_METHOD,
PROP_DO_CTTS,
};
/* some spare for header size as well */
......@@ -160,6 +203,7 @@ enum
#define DEFAULT_MOOV_RECOV_FILE NULL
#define DEFAULT_FRAGMENT_DURATION 0
#define DEFAULT_STREAMABLE FALSE
#define DEFAULT_DTS_METHOD DTS_METHOD_DD
static void gst_qt_mux_finalize (GObject * object);
......@@ -264,6 +308,11 @@ gst_qt_mux_class_init (GstQTMuxClass * klass)
"(in addition to decoding time) (use with caution)",
DEFAULT_DO_CTTS,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_DTS_METHOD,
g_param_spec_enum ("dts-method", "dts-method",
"Method to determine DTS time",
GST_TYPE_QT_MUX_DTS_METHOD, DEFAULT_DTS_METHOD,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_FAST_START,
g_param_spec_boolean ("faststart", "Format file to faststart",
"If the file should be formated for faststart (headers first). ",
......@@ -304,6 +353,8 @@ gst_qt_mux_class_init (GstQTMuxClass * klass)
static void
gst_qt_mux_pad_reset (GstQTPad * qtpad)
{
gint i;
qtpad->fourcc = 0;
qtpad->is_out_of_order = FALSE;
qtpad->have_dts = FALSE;
......@@ -314,6 +365,16 @@ gst_qt_mux_pad_reset (GstQTPad * qtpad)
qtpad->prepare_buf_func = NULL;
qtpad->avg_bitrate = 0;
qtpad->max_bitrate = 0;
qtpad->ts_n_entries = 0;
qtpad->buf_head = 0;
qtpad->buf_tail = 0;
for (i = 0; i < G_N_ELEMENTS (qtpad->buf_entries); i++) {
if (qtpad->buf_entries[i]) {
gst_buffer_unref (qtpad->buf_entries[i]);
qtpad->buf_entries[i] = NULL;
}
}
if (qtpad->last_buf)
gst_buffer_replace (&qtpad->last_buf, NULL);
......@@ -1930,12 +1991,50 @@ init:
return ret;
}
/* check whether @a differs from @b by order of @magn */
static gboolean inline
gst_qtmux_check_difference (GstQTMux * qtmux, GstClockTime a,
GstClockTime b, GstClockTime magn)
/* sigh, tiny list helpers to re-order stuff */
static void
gst_qt_mux_push_ts (GstQTMux * qtmux, GstQTPad * pad, GstClockTime ts)
{
gint i;
for (i = 0; (i < QTMUX_NO_OF_TS) && (i < pad->ts_n_entries); i++) {
if (ts > pad->ts_entries[i])
break;
}
memmove (&pad->ts_entries[i + 1], &pad->ts_entries[i],
sizeof (GstClockTime) * (pad->ts_n_entries - i));
pad->ts_entries[i] = ts;
pad->ts_n_entries++;
}
/* takes ownership of @buf */
static GstBuffer *
gst_qt_mux_get_asc_buffer_ts (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
{
return ((a >= b) ? (a - b >= (magn >> 1)) : (b - a >= (magn >> 1)));
const gint wrap = G_N_ELEMENTS (pad->buf_entries);
GstClockTime ts;
/* store buffer and ts, latter ordered */
if (buf) {
pad->buf_entries[pad->buf_tail++] = buf;
pad->buf_tail %= wrap;
gst_qt_mux_push_ts (qtmux, pad, GST_BUFFER_TIMESTAMP (buf));
}
if (pad->ts_n_entries && (!buf || pad->ts_n_entries >= QTMUX_NO_OF_TS)) {
ts = pad->ts_entries[--pad->ts_n_entries];
buf = pad->buf_entries[pad->buf_head];
pad->buf_entries[pad->buf_head++] = NULL;
pad->buf_head %= wrap;
buf = gst_buffer_make_metadata_writable (buf);
GST_BUFFER_TIMESTAMP (buf) = ts;
GST_DEBUG_OBJECT (qtmux, "next buffer uses reordered ts %" GST_TIME_FORMAT,
GST_TIME_ARGS (ts));
} else {
buf = NULL;
}
return buf;
}
/*
......@@ -1947,10 +2046,12 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
GstBuffer *last_buf = NULL;
GstClockTime duration;
guint nsamples, sample_size;
guint64 scaled_duration, chunk_offset;
gint64 last_dts;
guint64 chunk_offset;
gint64 last_dts, scaled_duration;
gint64 pts_offset = 0;
gboolean sync = FALSE, do_pts = FALSE;
gboolean drain = (buf == NULL);
GstFlowReturn ret;
if (!pad->fourcc)
goto not_negotiated;
......@@ -1960,6 +2061,15 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
buf = pad->prepare_buf_func (pad, buf, qtmux);
}
again:
if (G_UNLIKELY (qtmux->dts_method == DTS_METHOD_REORDER)) {
buf = gst_qt_mux_get_asc_buffer_ts (qtmux, pad, buf);
if (!buf) {
GST_DEBUG_OBJECT (qtmux, "no reordered buffer yet");
return GST_FLOW_OK;
}
}
last_buf = pad->last_buf;
if (last_buf == NULL) {
#ifndef GST_DISABLE_GST_DEBUG
......@@ -1979,16 +2089,29 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
gst_buffer_ref (last_buf);
/* nasty heuristic mess to guestimate dealing with DTS/PTS,
* while also trying to stay close to input ts to preserve sync, so:
* while also trying to stay close to input ts to preserve sync,
* so in DTS_METHOD_DD:
* - prefer using input ts where possible
* - if those detected out-of-order (*), and input duration available,
* mark as out-of-order and fallback to duration
* - if in out-of-order, need to preserve sync between streams, and adding
* durations might drift, so try to resync when we expect
* input ts == (sum of durations), which is at some keyframe input frame.
* - if those detected out-of-order (*), mark as out-of-order
* - if in out-of-order, then
* - if duration available, use that as delta
* Also mind to preserve sync between streams, and adding
* durations might drift, so try to resync when we expect
* input ts == (sum of durations), which is at some keyframe input frame.
* - if no duration available, we are actually in serious trouble and need
* to hack around that, so we fail.
* To remedy failure, alternatively, in DTS_METHOD_REORDER:
* - collect some buffers and re-order timestamp,
* then process the oldest buffer with smallest timestamps.
* This should typically compensate for some codec's handywork with ts.
* ... but in case this makes ts end up where not expected:
* - keep each ts with its buffer and still keep a list of most recent X ts,
* use the (ascending) minimum of those as DTS (and the difference as ts delta),
* and use this DTS as a basis to obtain a (positive) CTS offset.
* This should yield exact PTS == buffer ts, but it seems not all players
* out there are aware of ctts pts ...
*
* (*) if input ts out-of-order, or if ts differs from (sum of durations)
* by an (approx) order-of-duration magnitude
* 0.11 Phew, can we (pretty) please please sort out DTS/PTS on buffers ...
*/
if (G_LIKELY (buf) && !pad->is_out_of_order) {
if (G_LIKELY (GST_BUFFER_TIMESTAMP_IS_VALID (last_buf) &&
......@@ -2005,9 +2128,16 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
}
}
/* would have to be some unusual input, but not impossible */
if (G_UNLIKELY (qtmux->dts_method == DTS_METHOD_REORDER &&
pad->is_out_of_order)) {
goto no_order;
}
/* fall back to duration if last buffer or
* out-of-order (determined previously), otherwise use input ts */
if (buf == NULL || pad->is_out_of_order) {
if (buf == NULL ||
(pad->is_out_of_order && qtmux->dts_method == DTS_METHOD_DD)) {
if (!GST_BUFFER_DURATION_IS_VALID (last_buf)) {
/* be forgiving for some possibly last upstream flushed buffer */
if (buf)
......@@ -2020,9 +2150,7 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
/* avoid drift in sum timestamps,
* so use input timestamp for suitable keyframe */
if (buf && !GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT) &&
GST_BUFFER_TIMESTAMP (buf) >= pad->last_dts &&
!gst_qtmux_check_difference (qtmux, pad->last_dts + duration,
GST_BUFFER_TIMESTAMP (buf), duration)) {
GST_BUFFER_TIMESTAMP (buf) >= pad->last_dts) {
GST_DEBUG_OBJECT (qtmux, "resyncing out-of-order input to ts; "
"replacing %" GST_TIME_FORMAT " by %" GST_TIME_FORMAT,
GST_TIME_ARGS (pad->last_dts + duration),
......@@ -2030,8 +2158,37 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
duration = GST_BUFFER_TIMESTAMP (buf) - pad->last_dts;
}
}
} else {
} else if (qtmux->dts_method != DTS_METHOD_ASC) {
duration = GST_BUFFER_TIMESTAMP (buf) - GST_BUFFER_TIMESTAMP (last_buf);
} else {
GstClockTime ts;
g_assert (qtmux->dts_method == DTS_METHOD_ASC);
if (!qtmux->guess_pts)
goto need_pts;
/* add timestamp to queue; keeps in descending order */
gst_qt_mux_push_ts (qtmux, pad, GST_BUFFER_TIMESTAMP (last_buf));
/* chuck out smallest/last one if we have enough */
if (G_LIKELY (pad->ts_n_entries > QTMUX_NO_OF_TS))
pad->ts_n_entries--;
/* peek the now smallest timestamp */
ts = pad->ts_entries[pad->ts_n_entries - 1];
/* these tails are expected to be (strictly) ascending with
* large enough history */
GST_DEBUG_OBJECT (qtmux, "ASC method; base timestamp %" GST_TIME_FORMAT,
GST_TIME_ARGS (ts));
if (ts >= pad->last_dts) {
duration = ts - pad->last_dts;
} else {
/* fallback to previous value, negative ct offset might handle */
GST_WARNING_OBJECT (qtmux, "unexpected decrease in timestamp");
duration = 0;
}
/* arrange for small non-zero duration/delta << expected frame time */
ts = gst_util_uint64_scale (10, GST_SECOND,
atom_trak_get_timescale (pad->trak));
duration = MAX (duration, ts);
}
gst_buffer_replace (&pad->last_buf, buf);
......@@ -2166,15 +2323,23 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf)
if (qtmux->fragment_sequence) {
/* ensure that always sync samples are marked as such */
return gst_qt_mux_pad_fragment_add_buffer (qtmux, pad, last_buf,
buf == NULL, nsamples, last_dts, scaled_duration, sample_size,
ret = gst_qt_mux_pad_fragment_add_buffer (qtmux, pad, last_buf,
buf == NULL, nsamples, last_dts, (gint32) scaled_duration, sample_size,
!pad->sync || sync, pts_offset);
} else {
atom_trak_add_samples (pad->trak, nsamples, scaled_duration,
atom_trak_add_samples (pad->trak, nsamples, (gint32) scaled_duration,
sample_size, chunk_offset, sync, pts_offset);
return gst_qt_mux_send_buffer (qtmux, last_buf, &qtmux->mdat_size, TRUE);
ret = gst_qt_mux_send_buffer (qtmux, last_buf, &qtmux->mdat_size, TRUE);
}
if (G_UNLIKELY (drain && qtmux->dts_method == DTS_METHOD_REORDER &&
ret == GST_FLOW_OK)) {
buf = NULL;
goto again;
}
return ret;
/* ERRORS */
bail:
{
......@@ -2189,6 +2354,18 @@ no_time:
("Received buffer without timestamp/duration."));
goto bail;
}
no_order:
{
GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL),
("DTS method failed to re-order timestamps."));
goto bail;
}
need_pts:
{
GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL),
("Selected DTS method also needs PTS enabled."));
goto bail;
}
fragmented_sample:
{
GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL),
......@@ -3087,6 +3264,9 @@ gst_qt_mux_get_property (GObject * object,
case PROP_DO_CTTS:
g_value_set_boolean (value, qtmux->guess_pts);
break;
case PROP_DTS_METHOD:
g_value_set_enum (value, qtmux->dts_method);
break;
case PROP_FAST_START:
g_value_set_boolean (value, qtmux->fast_start);
break;
......@@ -3139,6 +3319,9 @@ gst_qt_mux_set_property (GObject * object,
case PROP_DO_CTTS:
qtmux->guess_pts = g_value_get_boolean (value);
break;
case PROP_DTS_METHOD:
qtmux->dts_method = g_value_get_enum (value);
break;
case PROP_FAST_START:
qtmux->fast_start = g_value_get_boolean (value);
break;
......
......@@ -78,6 +78,8 @@ typedef struct _GstQTPad GstQTPad;
typedef GstBuffer * (*GstQTPadPrepareBufferFunc) (GstQTPad * pad,
GstBuffer * buf, GstQTMux * qtmux);
#define QTMUX_NO_OF_TS 10
struct _GstQTPad
{
GstCollectData collect; /* we extend the CollectData */
......@@ -102,6 +104,11 @@ struct _GstQTPad
/* store the first timestamp for comparing with other streams and
* know if there are late streams */
GstClockTime first_ts;
GstClockTime ts_entries[QTMUX_NO_OF_TS + 2];
guint ts_n_entries;
GstBuffer *buf_entries[QTMUX_NO_OF_TS + 2];
guint buf_head;
guint buf_tail;
/* all the atom and chunk book-keeping is delegated here
* unowned/uncounted reference, parent MOOV owns */
......@@ -174,6 +181,7 @@ struct _GstQTMux
AtomsTreeFlavor flavor;
gboolean fast_start;
gboolean guess_pts;
gint dts_method;
gchar *fast_start_file_path;
gchar *moov_recov_file_path;
guint32 fragment_duration;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment