qtdemux.c 453 KB
Newer Older
1 2
/* GStreamer
 * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
3
 * Copyright (C) <2003> David A. Schleef <ds@schleef.org>
4
 * Copyright (C) <2006> Wim Taymans <wim@fluendo.com>
5
 * Copyright (C) <2007> Julien Moutte <julien@fluendo.com>
6
 * Copyright (C) <2009> Tim-Philipp Müller <tim centricular net>
7
 * Copyright (C) <2009> STEricsson <benjamin.gaignard@stericsson.com>
8
 * Copyright (C) <2013> Sreerenj Balachandran <sreerenj.balachandran@intel.com>
9 10
 * Copyright (C) <2013> Intel Corporation
 * Copyright (C) <2014> Centricular Ltd
11
 * Copyright (C) <2015> YouView TV Ltd.
12
 * Copyright (C) <2016> British Broadcasting Corporation
13
 *
14 15 16 17 18 19 20 21 22 23 24 25
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
Tim-Philipp Müller's avatar
Tim-Philipp Müller committed
26 27
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
28 29
 */

30 31 32 33
/**
 * SECTION:element-qtdemux
 *
 * Demuxes a .mov file into raw or compressed audio and/or video streams.
34
 *
35 36
 * This element supports both push and pull-based scheduling, depending on the
 * capabilities of the upstream elements.
37 38
 *
 * <refsect2>
39
 * <title>Example launch line</title>
40
 * |[
41
 * gst-launch-1.0 filesrc location=test.mov ! qtdemux name=demux  demux.audio_0 ! queue ! decodebin ! audioconvert ! audioresample ! autoaudiosink   demux.video_0 ! queue ! decodebin ! videoconvert ! videoscale ! autovideosink
42
 * ]| Play (parse and decode) a .mov file and try to output it to
43 44 45 46 47 48
 * an automatically detected soundcard and videosink. If the MOV file contains
 * compressed audio or video data, this will only work if you have the
 * right decoder elements/plugins installed.
 * </refsect2>
 */

49 50 51
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
52 53 54

#include "gst/gst-i18n-plugin.h"

55
#include <glib/gprintf.h>
56
#include <gst/tag/tag.h>
57
#include <gst/audio/audio.h>
58
#include <gst/video/video.h>
59

60
#include "qtatomparser.h"
61 62
#include "qtdemux_types.h"
#include "qtdemux_dump.h"
63
#include "fourcc.h"
64
#include "descriptors.h"
65
#include "qtdemux_lang.h"
66
#include "qtdemux.h"
67
#include "qtpalette.h"
68

69 70 71
#include "gst/riff/riff-media.h"
#include "gst/riff/riff-read.h"

72 73
#include <gst/pbutils/pbutils.h>

74
#include <stdio.h>
75
#include <stdlib.h>
76
#include <string.h>
77

78 79 80
#include <math.h>
#include <gst/math-compat.h>

81 82 83
#ifdef HAVE_ZLIB
# include <zlib.h>
#endif
84

85 86 87
/* max. size considered 'sane' for non-mdat atoms */
#define QTDEMUX_MAX_ATOM_SIZE (25*1024*1024)

88 89 90
/* if the sample index is larger than this, something is likely wrong */
#define QTDEMUX_MAX_SAMPLE_INDEX_SIZE (50*1024*1024)

91 92 93 94 95 96
/* For converting qt creation times to unix epoch times */
#define QTDEMUX_SECONDS_PER_DAY (60 * 60 * 24)
#define QTDEMUX_LEAP_YEARS_FROM_1904_TO_1970 17
#define QTDEMUX_SECONDS_FROM_1904_TO_1970 (((1970 - 1904) * (guint64) 365 + \
    QTDEMUX_LEAP_YEARS_FROM_1904_TO_1970) * QTDEMUX_SECONDS_PER_DAY)

97 98
#define QTDEMUX_TREE_NODE_FOURCC(n) (QT_FOURCC(((guint8 *) (n)->data) + 4))

99
#define STREAM_IS_EOS(s) (s->time_position == GST_CLOCK_TIME_NONE)
100

101 102
#define ABSDIFF(x, y) ( (x) > (y) ? ((x) - (y)) : ((y) - (x)) )

103
GST_DEBUG_CATEGORY (qtdemux_debug);
104

105
typedef struct _QtDemuxSegment QtDemuxSegment;
106
typedef struct _QtDemuxSample QtDemuxSample;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
107

108 109
typedef struct _QtDemuxCencSampleSetInfo QtDemuxCencSampleSetInfo;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
110 111
struct _QtDemuxSample
{
112
  guint32 size;
113
  gint32 pts_offset;            /* Add this value to timestamp to get the pts */
114
  guint64 offset;
115 116
  guint64 timestamp;            /* DTS In mov time */
  guint32 duration;             /* In mov time */
117
  gboolean keyframe;            /* TRUE when this packet is a keyframe */
118 119
};

120 121 122 123 124 125 126
/* Macros for converting to/from timescale */
#define QTSTREAMTIME_TO_GSTTIME(stream, value) (gst_util_uint64_scale((value), GST_SECOND, (stream)->timescale))
#define GSTTIME_TO_QTSTREAMTIME(stream, value) (gst_util_uint64_scale((value), (stream)->timescale, GST_SECOND))

#define QTTIME_TO_GSTTIME(qtdemux, value) (gst_util_uint64_scale((value), GST_SECOND, (qtdemux)->timescale))
#define GSTTIME_TO_QTTIME(qtdemux, value) (gst_util_uint64_scale((value), (qtdemux)->timescale, GST_SECOND))

127
/* timestamp is the DTS */
128
#define QTSAMPLE_DTS(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp))
129
/* timestamp + offset + cslg_shift is the outgoing PTS */
Nicolas Dufresne's avatar
Nicolas Dufresne committed
130
#define QTSAMPLE_PTS(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp + (stream)->cslg_shift + (sample)->pts_offset))
131 132
/* timestamp + offset is the PTS used for internal seek calcuations */
#define QTSAMPLE_PTS_NO_CSLG(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp + (sample)->pts_offset))
133
/* timestamp + duration - dts is the duration */
134
#define QTSAMPLE_DUR_DTS(stream, sample, dts) (QTSTREAMTIME_TO_GSTTIME ((stream), (sample)->timestamp + (sample)->duration) - (dts))
135 136 137

#define QTSAMPLE_KEYFRAME(stream,sample) ((stream)->all_keyframe || (sample)->keyframe)

138 139 140 141 142 143 144 145 146 147 148
/*
 * Quicktime has tracks and segments. A track is a continuous piece of
 * multimedia content. The track is not always played from start to finish but
 * instead, pieces of the track are 'cut out' and played in sequence. This is
 * what the segments do.
 *
 * Inside the track we have keyframes (K) and delta frames. The track has its
 * own timing, which starts from 0 and extends to end. The position in the track
 * is called the media_time.
 *
 * The segments now describe the pieces that should be played from this track
149
 * and are basically tuples of media_time/duration/rate entries. We can have
150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
 * multiple segments and they are all played after one another. An example:
 *
 * segment 1: media_time: 1 second, duration: 1 second, rate 1
 * segment 2: media_time: 3 second, duration: 2 second, rate 2
 *
 * To correctly play back this track, one must play: 1 second of media starting
 * from media_time 1 followed by 2 seconds of media starting from media_time 3
 * at a rate of 2.
 *
 * Each of the segments will be played at a specific time, the first segment at
 * time 0, the second one after the duration of the first one, etc.. Note that
 * the time in resulting playback is not identical to the media_time of the
 * track anymore.
 *
 * Visually, assuming the track has 4 second of media_time:
 *
 *                (a)                   (b)          (c)              (d)
 *         .-----------------------------------------------------------.
 * track:  | K.....K.........K........K.......K.......K...........K... |
 *         '-----------------------------------------------------------'
Stefan Kost's avatar
Stefan Kost committed
170
 *         0              1              2              3              4
171 172 173 174 175 176
 *           .------------^              ^   .----------^              ^
 *          /              .-------------'  /       .------------------'
 *         /              /          .-----'       /
 *         .--------------.         .--------------.
 *         | segment 1    |         | segment 2    |
 *         '--------------'         '--------------'
Stefan Kost's avatar
Stefan Kost committed
177
 *
178
 * The challenge here is to cut out the right pieces of the track for each of
179 180
 * the playback segments. This fortunately can easily be done with the SEGMENT
 * events of GStreamer.
181 182 183 184 185 186 187 188
 *
 * For playback of segment 1, we need to provide the decoder with the keyframe
 * (a), in the above figure, but we must instruct it only to output the decoded
 * data between second 1 and 2. We do this with a SEGMENT event for 1 to 2, time
 * position set to the time of the segment: 0.
 *
 * We then proceed to push data from keyframe (a) to frame (b). The decoder
 * decodes but clips all before media_time 1.
Stefan Kost's avatar
Stefan Kost committed
189
 *
190 191 192 193 194 195
 * After finishing a segment, we push out a new SEGMENT event with the clipping
 * boundaries of the new data.
 *
 * This is a good usecase for the GStreamer accumulated SEGMENT events.
 */

196 197 198
struct _QtDemuxSegment
{
  /* global time and duration, all gst time */
199 200 201
  GstClockTime time;
  GstClockTime stop_time;
  GstClockTime duration;
202
  /* media time of trak, all gst time */
203 204
  GstClockTime media_start;
  GstClockTime media_stop;
205
  gdouble rate;
206 207
  /* Media start time in trak timescale units */
  guint32 trak_media_start;
208 209
};

210 211
#define QTSEGMENT_IS_EMPTY(s) ((s)->media_start == GST_CLOCK_TIME_NONE)

212 213 214 215 216 217 218
/* Used with fragmented MP4 files (mfra atom) */
typedef struct
{
  GstClockTime ts;
  guint64 moof_offset;
} QtDemuxRandomAccessEntry;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
219 220
struct _QtDemuxStream
{
221 222 223
  GstPad *pad;

  /* stream type */
224 225
  guint32 subtype;
  GstCaps *caps;
226
  guint32 fourcc;
227
  gboolean sparse;
228

229 230 231 232
  gboolean new_caps;            /* If TRUE, caps need to be generated (by
                                 * calling _configure_stream()) This happens
                                 * for MSS and fragmented streams */

233
  gboolean new_stream;          /* signals that a stream_start is required */
234 235 236 237
  gboolean on_keyframe;         /* if this stream last pushed buffer was a
                                 * keyframe. This is important to identify
                                 * where to stop pushing buffers after a
                                 * segment stop time */
238

239 240 241
  /* if the stream has a redirect URI in its headers, we store it here */
  gchar *redirect_uri;

242 243 244
  /* track id */
  guint track_id;

245
  /* duration/scale */
246
  guint64 duration;             /* in timescale units */
247
  guint32 timescale;
248

249
  /* language */
250
  gchar lang_id[4];             /* ISO 639-2T language code */
251

252 253 254 255
  /* our samples */
  guint32 n_samples;
  QtDemuxSample *samples;
  gboolean all_keyframe;        /* TRUE when all samples are keyframes (no stss) */
256
  guint32 first_duration;       /* duration in timescale of first sample, used for figuring out
257
                                   the framerate */
258 259 260
  guint32 n_samples_moof;       /* sample count in a moof */
  guint64 duration_moof;        /* duration in timescale of a moof, used for figure out
                                 * the framerate of fragmented format stream */
261
  guint64 duration_last_moof;
262 263 264 265 266 267 268

  guint32 offset_in_sample;     /* Offset in the current sample, used for
                                 * streams which have got exceedingly big
                                 * sample size (such as 24s of raw audio).
                                 * Only used when max_buffer_size is non-NULL */
  guint32 max_buffer_size;      /* Maximum allowed size for output buffers.
                                 * Currently only set for raw audio streams*/
269

270 271
  /* if we use chunks or samples */
  gboolean sampled;
272
  guint padding;
273 274

  /* video info */
275 276
  gint width;
  gint height;
277 278 279 280 281
  /* aspect ratio */
  gint display_width;
  gint display_height;
  gint par_w;
  gint par_h;
282 283 284
  /* Numerator/denominator framerate */
  gint fps_n;
  gint fps_d;
285
  GstVideoColorimetry colorimetry;
286 287
  guint16 bits_per_sample;
  guint16 color_table_id;
288
  GstMemory *rgb8_palette;
289 290
  guint interlace_mode;
  guint field_order;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
291

292
  /* audio info */
293 294
  gdouble rate;
  gint n_channels;
295 296 297 298
  guint samples_per_packet;
  guint samples_per_frame;
  guint bytes_per_packet;
  guint bytes_per_sample;
299
  guint bytes_per_frame;
300
  guint compression;
301

302 303 304 305 306
  /* allocation */
  gboolean use_allocator;
  GstAllocator *allocator;
  GstAllocationParams params;

307 308
  gsize alignment;

309 310
  /* when a discontinuity is pending */
  gboolean discont;
311

312 313 314
  /* list of buffers to push first */
  GSList *buffers;

315 316 317 318
  /* if we need to clip this buffer. This is only needed for uncompressed
   * data */
  gboolean need_clip;

319 320 321
  /* buffer needs some custom processing, e.g. subtitles */
  gboolean need_process;

322 323 324
  /* current position */
  guint32 segment_index;
  guint32 sample_index;
325
  GstClockTime time_position;   /* in gst time */
326
  guint64 accumulated_base;
327

328 329 330
  /* the Gst segment we are processing out, used for clipping */
  GstSegment segment;

331 332 333
  /* quicktime segments */
  guint32 n_segments;
  QtDemuxSegment *segments;
334
  gboolean dummy_segment;
335 336
  guint32 from_sample;
  guint32 to_sample;
337 338

  gboolean sent_eos;
339
  GstTagList *stream_tags;
340
  gboolean send_global_tags;
341 342

  GstEvent *pending_event;
343 344 345 346 347 348 349 350 351

  GstByteReader stco;
  GstByteReader stsz;
  GstByteReader stsc;
  GstByteReader stts;
  GstByteReader stss;
  GstByteReader stps;
  GstByteReader ctts;

352
  gboolean chunks_are_samples;  /* TRUE means treat chunks as samples */
353
  gint64 stbl_index;
354 355
  /* stco */
  guint co_size;
356 357 358 359 360 361
  GstByteReader co_chunk;
  guint32 first_chunk;
  guint32 current_chunk;
  guint32 last_chunk;
  guint32 samples_per_chunk;
  guint32 stco_sample_index;
362 363 364
  /* stsz */
  guint32 sample_size;          /* 0 means variable sizes are stored in stsz */
  /* stsc */
365
  guint32 stsc_index;
366
  guint32 n_samples_per_chunk;
367 368 369
  guint32 stsc_chunk_index;
  guint32 stsc_sample_index;
  guint64 chunk_offset;
370
  /* stts */
371 372
  guint32 stts_index;
  guint32 stts_samples;
373
  guint32 n_sample_times;
374
  guint32 stts_sample_index;
375
  guint64 stts_time;
376
  guint32 stts_duration;
377
  /* stss */
378
  gboolean stss_present;
379
  guint32 n_sample_syncs;
380
  guint32 stss_index;
381
  /* stps */
382
  gboolean stps_present;
383
  guint32 n_sample_partial_syncs;
384
  guint32 stps_index;
385 386 387
  QtDemuxRandomAccessEntry *ra_entries;
  guint n_ra_entries;

388 389
  const QtDemuxRandomAccessEntry *pending_seek;

390 391 392
  /* ctts */
  gboolean ctts_present;
  guint32 n_composition_times;
393 394 395 396
  guint32 ctts_index;
  guint32 ctts_sample_index;
  guint32 ctts_count;
  gint32 ctts_soffset;
397

Nicolas Dufresne's avatar
Nicolas Dufresne committed
398 399 400
  /* cslg */
  guint32 cslg_shift;

401 402 403 404 405
  /* fragmented */
  gboolean parsed_trex;
  guint32 def_sample_duration;
  guint32 def_sample_size;
  guint32 def_sample_flags;
406 407

  gboolean disabled;
408 409 410 411

  /* stereoscopic video streams */
  GstVideoMultiviewMode multiview_mode;
  GstVideoMultiviewFlags multiview_flags;
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428

  /* protected streams */
  gboolean protected;
  guint32 protection_scheme_type;
  guint32 protection_scheme_version;
  gpointer protection_scheme_info;      /* specific to the protection scheme */
  GQueue protection_scheme_event_queue;
};

/* Contains properties and cryptographic info for a set of samples from a
 * track protected using Common Encryption (cenc) */
struct _QtDemuxCencSampleSetInfo
{
  GstStructure *default_properties;

  /* @crypto_info holds one GstStructure per sample */
  GPtrArray *crypto_info;
429 430
};

431 432
static const gchar *
qt_demux_state_string (enum QtDemuxState state)
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
433
{
434 435 436 437 438 439 440 441 442 443 444 445 446
  switch (state) {
    case QTDEMUX_STATE_INITIAL:
      return "<INITIAL>";
    case QTDEMUX_STATE_HEADER:
      return "<HEADER>";
    case QTDEMUX_STATE_MOVIE:
      return "<MOVIE>";
    case QTDEMUX_STATE_BUFFER_MDAT:
      return "<BUFFER_MDAT>";
    default:
      return "<UNKNOWN>";
  }
}
447

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
448
static GNode *qtdemux_tree_get_child_by_type (GNode * node, guint32 fourcc);
449
static GNode *qtdemux_tree_get_child_by_type_full (GNode * node,
450
    guint32 fourcc, GstByteReader * parser);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
451
static GNode *qtdemux_tree_get_sibling_by_type (GNode * node, guint32 fourcc);
452 453
static GNode *qtdemux_tree_get_sibling_by_type_full (GNode * node,
    guint32 fourcc, GstByteReader * parser);
454

455 456
static GstFlowReturn qtdemux_add_fragmented_samples (GstQTDemux * qtdemux);

David Schleef's avatar
David Schleef committed
457
static GstStaticPadTemplate gst_qtdemux_sink_template =
458
    GST_STATIC_PAD_TEMPLATE ("sink",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
459
    GST_PAD_SINK,
460
    GST_PAD_ALWAYS,
461 462
    GST_STATIC_CAPS ("video/quicktime; video/mj2; audio/x-m4a; "
        "application/x-3gp")
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
463
    );
David Schleef's avatar
David Schleef committed
464 465

static GstStaticPadTemplate gst_qtdemux_videosrc_template =
Wim Taymans's avatar
Wim Taymans committed
466
GST_STATIC_PAD_TEMPLATE ("video_%u",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
467 468 469
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);
David Schleef's avatar
David Schleef committed
470 471

static GstStaticPadTemplate gst_qtdemux_audiosrc_template =
Wim Taymans's avatar
Wim Taymans committed
472
GST_STATIC_PAD_TEMPLATE ("audio_%u",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
473 474 475
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);
476

477
static GstStaticPadTemplate gst_qtdemux_subsrc_template =
Wim Taymans's avatar
Wim Taymans committed
478
GST_STATIC_PAD_TEMPLATE ("subtitle_%u",
479 480 481 482
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
483 484
#define gst_qtdemux_parent_class parent_class
G_DEFINE_TYPE (GstQTDemux, gst_qtdemux, GST_TYPE_ELEMENT);
485

486
static void gst_qtdemux_dispose (GObject * object);
487

488 489
static guint32
gst_qtdemux_find_index_linear (GstQTDemux * qtdemux, QtDemuxStream * str,
490
    GstClockTime media_time);
491 492 493 494
static guint32
gst_qtdemux_find_index_for_given_media_offset_linear (GstQTDemux * qtdemux,
    QtDemuxStream * str, gint64 media_offset);

495
#if 0
496 497
static void gst_qtdemux_set_index (GstElement * element, GstIndex * index);
static GstIndex *gst_qtdemux_get_index (GstElement * element);
498
#endif
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
499 500
static GstStateChangeReturn gst_qtdemux_change_state (GstElement * element,
    GstStateChange transition);
501
static gboolean qtdemux_sink_activate (GstPad * sinkpad, GstObject * parent);
Wim Taymans's avatar
Wim Taymans committed
502 503
static gboolean qtdemux_sink_activate_mode (GstPad * sinkpad,
    GstObject * parent, GstPadMode mode, gboolean active);
504 505

static void gst_qtdemux_loop (GstPad * pad);
Wim Taymans's avatar
Wim Taymans committed
506 507 508 509
static GstFlowReturn gst_qtdemux_chain (GstPad * sinkpad, GstObject * parent,
    GstBuffer * inbuf);
static gboolean gst_qtdemux_handle_sink_event (GstPad * pad, GstObject * parent,
    GstEvent * event);
Thiago Santos's avatar
Thiago Santos committed
510
static gboolean gst_qtdemux_setcaps (GstQTDemux * qtdemux, GstCaps * caps);
511 512
static gboolean gst_qtdemux_configure_stream (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
513 514
static GstFlowReturn gst_qtdemux_process_adapter (GstQTDemux * demux,
    gboolean force);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
515

516 517
static gboolean qtdemux_parse_moov (GstQTDemux * qtdemux,
    const guint8 * buffer, guint length);
518
static gboolean qtdemux_parse_node (GstQTDemux * qtdemux, GNode * node,
519
    const guint8 * buffer, guint length);
520
static gboolean qtdemux_parse_tree (GstQTDemux * qtdemux);
Thiago Santos's avatar
Thiago Santos committed
521 522
static void qtdemux_parse_udta (GstQTDemux * qtdemux, GstTagList * taglist,
    GNode * udta);
523

524
static void gst_qtdemux_handle_esds (GstQTDemux * qtdemux,
525
    QtDemuxStream * stream, GNode * esds, GstTagList * list);
526 527
static GstCaps *qtdemux_video_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * stsd_data,
528
    gchar ** codec_name);
529 530
static GstCaps *qtdemux_audio_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * data, int len,
531
    gchar ** codec_name);
532
static GstCaps *qtdemux_sub_caps (GstQTDemux * qtdemux,
533 534
    QtDemuxStream * stream, guint32 fourcc, const guint8 * data,
    gchar ** codec_name);
535 536 537 538
static GstCaps *qtdemux_generic_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * stsd_data,
    gchar ** codec_name);

539 540
static gboolean qtdemux_parse_samples (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 n);
541
static GstFlowReturn qtdemux_expose_streams (GstQTDemux * qtdemux);
542 543
static void gst_qtdemux_stream_free (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Thiago Santos's avatar
Thiago Santos committed
544 545
static void gst_qtdemux_stream_clear (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Arnaud Vrac's avatar
Arnaud Vrac committed
546
static void gst_qtdemux_remove_stream (GstQTDemux * qtdemux, int index);
547
static GstFlowReturn qtdemux_prepare_streams (GstQTDemux * qtdemux);
548 549
static void qtdemux_do_allocation (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
550 551 552 553 554
static gboolean gst_qtdemux_activate_segment (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 seg_idx, GstClockTime offset);
static gboolean gst_qtdemux_stream_update_segment (GstQTDemux * qtdemux,
    QtDemuxStream * stream, gint seg_idx, GstClockTime offset,
    GstClockTime * _start, GstClockTime * _stop);
555 556
static void gst_qtdemux_send_gap_for_segment (GstQTDemux * demux,
    QtDemuxStream * stream, gint segment_index, GstClockTime pos);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
557

558
static gboolean qtdemux_pull_mfro_mfra (GstQTDemux * qtdemux);
559
static void check_update_duration (GstQTDemux * qtdemux, GstClockTime duration);
560

561 562 563 564
static gchar *qtdemux_uuid_bytes_to_string (gconstpointer uuid_bytes);

static GstStructure *qtdemux_get_cenc_sample_properties (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint sample_index);
565 566
static void gst_qtdemux_append_protection_system_id (GstQTDemux * qtdemux,
    const gchar * id);
567
static void qtdemux_gst_structure_free (GstStructure * gststructure);
568

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
569 570
static void
gst_qtdemux_class_init (GstQTDemuxClass * klass)
571 572 573 574
{
  GObjectClass *gobject_class;
  GstElementClass *gstelement_class;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
575 576
  gobject_class = (GObjectClass *) klass;
  gstelement_class = (GstElementClass *) klass;
577

578
  parent_class = g_type_class_peek_parent (klass);
579

580 581
  gobject_class->dispose = gst_qtdemux_dispose;

582
  gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_qtdemux_change_state);
583
#if 0
584 585
  gstelement_class->set_index = GST_DEBUG_FUNCPTR (gst_qtdemux_set_index);
  gstelement_class->get_index = GST_DEBUG_FUNCPTR (gst_qtdemux_get_index);
586
#endif
587 588

  gst_tag_register_musicbrainz_tags ();
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
589

590 591 592 593 594 595 596 597
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_sink_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_videosrc_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_audiosrc_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_subsrc_template);
598
  gst_element_class_set_static_metadata (gstelement_class, "QuickTime demuxer",
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
599 600 601 602 603 604
      "Codec/Demuxer",
      "Demultiplex a QuickTime file into audio and video streams",
      "David Schleef <ds@schleef.org>, Wim Taymans <wim@fluendo.com>");

  GST_DEBUG_CATEGORY_INIT (qtdemux_debug, "qtdemux", 0, "qtdemux plugin");

605 606
}

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
607
static void
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
608
gst_qtdemux_init (GstQTDemux * qtdemux)
609
{
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
610
  qtdemux->sinkpad =
611
      gst_pad_new_from_static_template (&gst_qtdemux_sink_template, "sink");
612
  gst_pad_set_activate_function (qtdemux->sinkpad, qtdemux_sink_activate);
Wim Taymans's avatar
Wim Taymans committed
613 614
  gst_pad_set_activatemode_function (qtdemux->sinkpad,
      qtdemux_sink_activate_mode);
615 616
  gst_pad_set_chain_function (qtdemux->sinkpad, gst_qtdemux_chain);
  gst_pad_set_event_function (qtdemux->sinkpad, gst_qtdemux_handle_sink_event);
617
  gst_element_add_pad (GST_ELEMENT_CAST (qtdemux), qtdemux->sinkpad);
618

619 620
  qtdemux->state = QTDEMUX_STATE_INITIAL;
  qtdemux->pullbased = FALSE;
621
  qtdemux->posted_redirect = FALSE;
622 623 624
  qtdemux->neededbytes = 16;
  qtdemux->todrop = 0;
  qtdemux->adapter = gst_adapter_new ();
625
  qtdemux->offset = 0;
626
  qtdemux->first_mdat = -1;
627
  qtdemux->got_moov = FALSE;
628
  qtdemux->mdatoffset = -1;
629
  qtdemux->mdatbuffer = NULL;
630
  qtdemux->restoredata_buffer = NULL;
631
  qtdemux->restoredata_offset = -1;
632
  qtdemux->fragment_start = -1;
633
  qtdemux->fragment_start_offset = -1;
634 635 636
  qtdemux->media_caps = NULL;
  qtdemux->exposed = FALSE;
  qtdemux->mss_mode = FALSE;
637
  qtdemux->pending_newsegment = NULL;
638
  qtdemux->upstream_format_is_time = FALSE;
639 640
  qtdemux->have_group_id = FALSE;
  qtdemux->group_id = G_MAXUINT;
641 642 643
  qtdemux->cenc_aux_info_offset = 0;
  qtdemux->cenc_aux_info_sizes = NULL;
  qtdemux->cenc_aux_sample_count = 0;
644 645
  qtdemux->protection_system_ids = NULL;
  g_queue_init (&qtdemux->protection_event_queue);
646
  gst_segment_init (&qtdemux->segment, GST_FORMAT_TIME);
647 648
  qtdemux->tag_list = gst_tag_list_new_empty ();
  gst_tag_list_set_scope (qtdemux->tag_list, GST_TAG_SCOPE_GLOBAL);
Thiago Santos's avatar
Thiago Santos committed
649
  qtdemux->flowcombiner = gst_flow_combiner_new ();
Wim Taymans's avatar
Wim Taymans committed
650 651

  GST_OBJECT_FLAG_SET (qtdemux, GST_ELEMENT_FLAG_INDEXABLE);
652 653
}

654 655 656 657 658 659 660 661 662
static void
gst_qtdemux_dispose (GObject * object)
{
  GstQTDemux *qtdemux = GST_QTDEMUX (object);

  if (qtdemux->adapter) {
    g_object_unref (G_OBJECT (qtdemux->adapter));
    qtdemux->adapter = NULL;
  }
663
  gst_tag_list_unref (qtdemux->tag_list);
Thiago Santos's avatar
Thiago Santos committed
664
  gst_flow_combiner_free (qtdemux->flowcombiner);
665 666 667
  g_queue_foreach (&qtdemux->protection_event_queue, (GFunc) gst_event_unref,
      NULL);
  g_queue_clear (&qtdemux->protection_event_queue);
668

669 670 671
  g_free (qtdemux->cenc_aux_info_sizes);
  qtdemux->cenc_aux_info_sizes = NULL;

672
  G_OBJECT_CLASS (parent_class)->dispose (object);
673 674
}

675 676 677 678
static void
gst_qtdemux_post_no_playable_stream_error (GstQTDemux * qtdemux)
{
  if (qtdemux->posted_redirect) {
679
    GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
680 681 682
        (_("This file contains no playable streams.")),
        ("no known streams found, a redirect message has been posted"));
  } else {
683
    GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
684 685 686 687 688
        (_("This file contains no playable streams.")),
        ("no known streams found"));
  }
}

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
689 690 691
static GstBuffer *
_gst_buffer_new_wrapped (gpointer mem, gsize size, GFreeFunc free_func)
{
Wim Taymans's avatar
Wim Taymans committed
692 693
  return gst_buffer_new_wrapped_full (free_func ? 0 : GST_MEMORY_FLAG_READONLY,
      mem, size, 0, size, mem, free_func);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
694 695
}

696 697 698 699 700
static GstFlowReturn
gst_qtdemux_pull_atom (GstQTDemux * qtdemux, guint64 offset, guint64 size,
    GstBuffer ** buf)
{
  GstFlowReturn flow;
Wim Taymans's avatar
Wim Taymans committed
701
  GstMapInfo map;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
702
  gsize bsize;
703

704
  if (G_UNLIKELY (size == 0)) {
705 706 707 708 709 710 711
    GstFlowReturn ret;
    GstBuffer *tmp = NULL;

    ret = gst_qtdemux_pull_atom (qtdemux, offset, sizeof (guint32), &tmp);
    if (ret != GST_FLOW_OK)
      return ret;

Wim Taymans's avatar
Wim Taymans committed
712 713
    gst_buffer_map (tmp, &map, GST_MAP_READ);
    size = QT_UINT32 (map.data);
714
    GST_DEBUG_OBJECT (qtdemux, "size 0x%08" G_GINT64_MODIFIER "x", size);
715

Wim Taymans's avatar
Wim Taymans committed
716
    gst_buffer_unmap (tmp, &map);
717 718 719
    gst_buffer_unref (tmp);
  }

720 721
  /* Sanity check: catch bogus sizes (fuzzed/broken files) */
  if (G_UNLIKELY (size > QTDEMUX_MAX_ATOM_SIZE)) {
722 723 724 725 726
    if (qtdemux->state != QTDEMUX_STATE_MOVIE && qtdemux->got_moov) {
      /* we're pulling header but already got most interesting bits,
       * so never mind the rest (e.g. tags) (that much) */
      GST_WARNING_OBJECT (qtdemux, "atom has bogus size %" G_GUINT64_FORMAT,
          size);
727
      return GST_FLOW_EOS;
728 729 730 731 732 733
    } else {
      GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
          (_("This file is invalid and cannot be played.")),
          ("atom has bogus size %" G_GUINT64_FORMAT, size));
      return GST_FLOW_ERROR;
    }
734 735 736 737 738 739 740
  }

  flow = gst_pad_pull_range (qtdemux->sinkpad, offset, size, buf);

  if (G_UNLIKELY (flow != GST_FLOW_OK))
    return flow;

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
741
  bsize = gst_buffer_get_size (*buf);
742
  /* Catch short reads - we don't want any partial atoms */
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
743
  if (G_UNLIKELY (bsize < size)) {
744 745
    GST_WARNING_OBJECT (qtdemux,
        "short read: %" G_GSIZE_FORMAT " < %" G_GUINT64_FORMAT, bsize, size);
746 747
    gst_buffer_unref (*buf);
    *buf = NULL;
748
    return GST_FLOW_EOS;
749 750 751 752 753
  }

  return flow;
}

754
#if 1
755
static gboolean
756 757 758
gst_qtdemux_src_convert (GstQTDemux * qtdemux, GstPad * pad,
    GstFormat src_format, gint64 src_value, GstFormat dest_format,
    gint64 * dest_value)
759 760
{
  gboolean res = TRUE;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
761
  QtDemuxStream *stream = gst_pad_get_element_private (pad);
762
  gint32 index;
763

Andy Wingo's avatar
Andy Wingo committed
764 765 766 767
  if (stream->subtype != FOURCC_vide) {
    res = FALSE;
    goto done;
  }
768 769 770

  switch (src_format) {
    case GST_FORMAT_TIME:
771 772 773
      switch (dest_format) {
        case GST_FORMAT_BYTES:{
          index = gst_qtdemux_find_index_linear (qtdemux, stream, src_value);
774 775 776 777
          if (-1 == index) {
            res = FALSE;
            goto done;
          }
778 779 780 781 782 783

          *dest_value = stream->samples[index].offset;

          GST_DEBUG_OBJECT (qtdemux, "Format Conversion Time->Offset :%"
              GST_TIME_FORMAT "->%" G_GUINT64_FORMAT,
              GST_TIME_ARGS (src_value), *dest_value);
784
          break;
785
        }
786 787 788
        default:
          res = FALSE;
          break;
789 790 791
      }
      break;
    case GST_FORMAT_BYTES:
792 793 794 795 796 797
      switch (dest_format) {
        case GST_FORMAT_TIME:{
          index =
              gst_qtdemux_find_index_for_given_media_offset_linear (qtdemux,
              stream, src_value);

798 799 800 801
          if (-1 == index) {
            res = FALSE;
            goto done;
          }
802 803

          *dest_value =
804 805 806 807 808
              QTSTREAMTIME_TO_GSTTIME (stream,
              stream->samples[index].timestamp);
          GST_DEBUG_OBJECT (qtdemux,
              "Format Conversion Offset->Time :%" G_GUINT64_FORMAT "->%"
              GST_TIME_FORMAT, src_value, GST_TIME_ARGS (*dest_value));
809
          break;
810
        }
811 812 813
        default:
          res = FALSE;
          break;
814 815 816 817
      }
      break;
    default:
      res = FALSE;
818
      break;
819 820
  }

Andy Wingo's avatar
Andy Wingo committed
821
done:
822 823
  return res;
}
824
#endif
825

826
static gboolean
827
gst_qtdemux_get_duration (GstQTDemux * qtdemux, GstClockTime * duration)
828
{
829
  gboolean res = FALSE;
830 831 832

  *duration = GST_CLOCK_TIME_NONE;

833 834 835 836 837 838
  if (qtdemux->duration != 0 &&
      qtdemux->duration != G_MAXINT64 && qtdemux->timescale != 0) {
    *duration = QTTIME_TO_GSTTIME (qtdemux, qtdemux->duration);
    res = TRUE;
  } else {
    *duration = GST_CLOCK_TIME_NONE;
839
  }
840

841 842 843
  return res;
}

844
static gboolean
Wim Taymans's avatar
Wim Taymans committed
845 846
gst_qtdemux_handle_src_query (GstPad * pad, GstObject * parent,
    GstQuery * query)
847
{
848
  gboolean res = FALSE;
Wim Taymans's avatar
Wim Taymans committed
849
  GstQTDemux *qtdemux = GST_QTDEMUX (parent);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
850

851 852
  GST_LOG_OBJECT (pad, "%s query", GST_QUERY_TYPE_NAME (query));

853
  switch (GST_QUERY_TYPE (query)) {
854 855 856 857 858 859
    case GST_QUERY_POSITION:{
      GstFormat fmt;

      gst_query_parse_position (query, &fmt, NULL);
      if (fmt == GST_FORMAT_TIME
          && GST_CLOCK_TIME_IS_VALID (qtdemux->segment.position)) {
860
        gst_query_set_position (query, GST_FORMAT_TIME,
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
861
            qtdemux->segment.position);
Wim Taymans's avatar
Wim Taymans committed
862 863
        res = TRUE;
      }
864
    }
Wim Taymans's avatar
Wim Taymans committed
865
      break;
866 867
    case GST_QUERY_DURATION:{
      GstFormat fmt;
868

869 870
      gst_query_parse_duration (query, &fmt, NULL);
      if (fmt == GST_FORMAT_TIME) {
871 872 873
        /* First try to query upstream */
        res = gst_pad_query_default (pad, parent, query);
        if (!res) {
874
          GstClockTime duration;
875
          if (gst_qtdemux_get_duration (qtdemux, &duration) && duration > 0) {
876 877 878
            gst_query_set_duration (query, GST_FORMAT_TIME, duration);
            res = TRUE;
          }
879 880 881
        }
      }
      break;
882
    }
883 884
    case GST_QUERY_CONVERT:{
      GstFormat src_fmt, dest_fmt;
885
      gint64 src_value, dest_value = 0;
886 887 888

      gst_query_parse_convert (query, &src_fmt, &src_value, &dest_fmt, NULL);

889
      res = gst_qtdemux_src_convert (qtdemux, pad,
890 891 892 893 894 895 896 897 898 899 900
          src_fmt, src_value, dest_fmt, &dest_value);
      if (res) {
        gst_query_set_convert (query, src_fmt, src_value, dest_fmt, dest_value);
        res = TRUE;
      }
      break;
    }
    case GST_QUERY_FORMATS:
      gst_query_set_formats (query, 2, GST_FORMAT_TIME, GST_FORMAT_BYTES);
      res = TRUE;
      break;
901 902
    case GST_QUERY_SEEKING:{
      GstFormat fmt;
903
      gboolean seekable;
904

Thiago Santos's avatar
Thiago Santos committed
905 906 907 908 909 910
      /* try upstream first */
      res = gst_pad_query_default (pad, parent, query);

      if (!res) {
        gst_query_parse_seeking (query, &fmt, NULL, NULL, NULL);
        if (fmt == GST_FORMAT_TIME) {
911
          GstClockTime duration;
Thiago Santos's avatar
Thiago Santos committed
912 913 914 915 916 917 918 919 920 921 922 923 924 925

          gst_qtdemux_get_duration (qtdemux, &duration);
          seekable = TRUE;
          if (!qtdemux->pullbased) {
            GstQuery *q;

            /* we might be able with help from upstream */
            seekable = FALSE;
            q = gst_query_new_seeking (GST_FORMAT_BYTES);
            if (gst_pad_peer_query (qtdemux->sinkpad, q)) {
              gst_query_parse_seeking (q, &fmt, &seekable, NULL, NULL);
              GST_LOG_OBJECT (qtdemux, "upstream BYTE seekable %d", seekable);
            }
            gst_query_unref (q);
926
          }
Thiago Santos's avatar
Thiago Santos committed
927 928
          gst_query_set_seeking (query, GST_FORMAT_TIME, seekable, 0, duration);
          res = TRUE;
929
        }
930
      }
931
      break;
932
    }
Wim Taymans's avatar
Wim Taymans committed
933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
    case GST_QUERY_SEGMENT:
    {
      GstFormat format;
      gint64 start, stop;

      format = qtdemux->segment.format;

      start =
          gst_segment_to_stream_time (&qtdemux->segment, format,
          qtdemux->segment.start);
      if ((stop = qtdemux->segment.stop) == -1)
        stop = qtdemux->segment.duration;
      else
        stop = gst_segment_to_stream_time (&qtdemux->segment, format, stop);

      gst_query_set_segment (query, qtdemux->segment.rate, format, start, stop);
      res = TRUE;
      break;
    }
952
    default:
Wim Taymans's avatar
Wim Taymans committed
953
      res = gst_pad_query_default (pad, parent, query);
954 955 956 957 958 959
      break;
  }

  return res;
}

960 961 962 963 964 965 966
static void
gst_qtdemux_push_tags (GstQTDemux * qtdemux, QtDemuxStream * stream)
{
  if (G_LIKELY (stream->pad)) {
    GST_DEBUG_OBJECT (qtdemux, "Checking pad %s:%s for tags",
        GST_DEBUG_PAD_NAME (stream->pad));

967
    if (!gst_tag_list_is_empty (stream->stream_tags)) {
968
      GST_DEBUG_OBJECT (qtdemux, "Sending tags %" GST_PTR_FORMAT,
969
          stream->stream_tags);
970
      gst_pad_push_event (stream->pad,
971
          gst_event_new_tag (gst_tag_list_ref (stream->stream_tags)));
972 973
    }

974
    if (G_UNLIKELY (stream->send_global_tags)) {
975 976 977
      GST_DEBUG_OBJECT (qtdemux, "Sending global tags %" GST_PTR_FORMAT,
          qtdemux->tag_list);
      gst_pad_push_event (stream->pad,
978
          gst_event_new_tag (gst_tag_list_ref (qtdemux->tag_list)));
979 980 981 982 983
      stream->send_global_tags = FALSE;
    }
  }
}

984
/* push event on all source pads; takes ownership of the event */
985
static void
986
gst_qtdemux_push_event (GstQTDemux * qtdemux, GstEvent * event)
987 988
{
  guint n;
989
  gboolean has_valid_stream = FALSE;
990
  GstEventType etype = GST_EVENT_TYPE (event);
991 992 993 994 995

  GST_DEBUG_OBJECT (qtdemux, "pushing %s event on all source pads",
      GST_EVENT_TYPE_NAME (event));

  for (n = 0; n < qtdemux->n_streams; n++) {
996
    GstPad *pad;
997
    QtDemuxStream *stream = qtdemux->streams[n];
Thiago Santos's avatar
Thiago Santos committed
998
    GST_DEBUG_OBJECT (qtdemux, "pushing on pad %i", n);
999

1000 1001 1002
    if ((pad = stream->pad)) {
      has_valid_stream = TRUE;

1003 1004 1005 1006
      if (etype == GST_EVENT_EOS) {
        /* let's not send twice */
        if (stream->sent_eos)
          continue;
1007
        stream->sent_eos = TRUE;
1008
      }
1009 1010

      gst_pad_push_event (pad, gst_event_ref (event));
1011
    }
1012
  }
1013

1014
  gst_event_unref (event);
1015

1016 1017
  /* if it is EOS and there are no pads, post an error */
  if (!has_valid_stream && etype == GST_EVENT_EOS) {
Thiago Santos's avatar