gsth264parse.c 92.2 KB
Newer Older
1
/* GStreamer H.264 Parser
2
 * Copyright (C) <2010> Collabora ltd
3
 * Copyright (C) <2010> Nokia Corporation
4 5 6 7
 * Copyright (C) <2011> Intel Corporation
 *
 * Copyright (C) <2010> Mark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>
 * Copyright (C) <2011> Thibault Saunier <thibault.saunier@collabora.com>
8 9 10 11 12 13 14 15 16 17 18 19 20
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
Tim-Philipp Müller's avatar
Tim-Philipp Müller committed
21 22
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
23 24 25 26 27 28
 */

#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

29 30
#include <gst/base/base.h>
#include <gst/pbutils/pbutils.h>
31
#include <gst/video/video.h>
32 33 34 35 36 37 38 39 40 41 42 43
#include "gsth264parse.h"

#include <string.h>

GST_DEBUG_CATEGORY (h264_parse_debug);
#define GST_CAT_DEFAULT h264_parse_debug

#define DEFAULT_CONFIG_INTERVAL      (0)

enum
{
  PROP_0,
44
  PROP_CONFIG_INTERVAL
45 46 47 48 49 50
};

enum
{
  GST_H264_PARSE_FORMAT_NONE,
  GST_H264_PARSE_FORMAT_AVC,
51 52
  GST_H264_PARSE_FORMAT_BYTE,
  GST_H264_PARSE_FORMAT_AVC3
53 54 55 56 57 58 59 60 61
};

enum
{
  GST_H264_PARSE_ALIGN_NONE = 0,
  GST_H264_PARSE_ALIGN_NAL,
  GST_H264_PARSE_ALIGN_AU
};

62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
enum
{
  GST_H264_PARSE_STATE_GOT_SPS = 1 << 0,
  GST_H264_PARSE_STATE_GOT_PPS = 1 << 1,
  GST_H264_PARSE_STATE_GOT_SLICE = 1 << 2,

  GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS = (GST_H264_PARSE_STATE_GOT_SPS |
      GST_H264_PARSE_STATE_GOT_PPS),
  GST_H264_PARSE_STATE_VALID_PICTURE =
      (GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS |
      GST_H264_PARSE_STATE_GOT_SLICE)
};

#define GST_H264_PARSE_STATE_VALID(parse, expected_state) \
  (((parse)->state & (expected_state)) == (expected_state))

78 79 80
static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink",
    GST_PAD_SINK,
    GST_PAD_ALWAYS,
81
    GST_STATIC_CAPS ("video/x-h264"));
82 83 84 85

static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
86
    GST_STATIC_CAPS ("video/x-h264, parsed = (boolean) true, "
87
        "stream-format=(string) { avc, avc3, byte-stream }, "
88
        "alignment=(string) { au, nal }"));
89

90 91
#define parent_class gst_h264_parse_parent_class
G_DEFINE_TYPE (GstH264Parse, gst_h264_parse, GST_TYPE_BASE_PARSE);
92 93 94 95 96

static void gst_h264_parse_finalize (GObject * object);

static gboolean gst_h264_parse_start (GstBaseParse * parse);
static gboolean gst_h264_parse_stop (GstBaseParse * parse);
97 98
static GstFlowReturn gst_h264_parse_handle_frame (GstBaseParse * parse,
    GstBaseParseFrame * frame, gint * skipsize);
99 100 101 102 103 104 105 106 107 108 109
static GstFlowReturn gst_h264_parse_parse_frame (GstBaseParse * parse,
    GstBaseParseFrame * frame);
static GstFlowReturn gst_h264_parse_pre_push_frame (GstBaseParse * parse,
    GstBaseParseFrame * frame);

static void gst_h264_parse_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_h264_parse_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);

static gboolean gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps);
110 111
static GstCaps *gst_h264_parse_get_caps (GstBaseParse * parse,
    GstCaps * filter);
112 113 114
static gboolean gst_h264_parse_event (GstBaseParse * parse, GstEvent * event);
static gboolean gst_h264_parse_src_event (GstBaseParse * parse,
    GstEvent * event);
115 116
static void gst_h264_parse_update_src_caps (GstH264Parse * h264parse,
    GstCaps * caps);
117 118 119 120 121 122

static void
gst_h264_parse_class_init (GstH264ParseClass * klass)
{
  GObjectClass *gobject_class = (GObjectClass *) klass;
  GstBaseParseClass *parse_class = GST_BASE_PARSE_CLASS (klass);
123 124 125
  GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass);

  GST_DEBUG_CATEGORY_INIT (h264_parse_debug, "h264parse", 0, "h264 parser");
126 127 128 129 130 131

  gobject_class->finalize = gst_h264_parse_finalize;
  gobject_class->set_property = gst_h264_parse_set_property;
  gobject_class->get_property = gst_h264_parse_get_property;

  g_object_class_install_property (gobject_class, PROP_CONFIG_INTERVAL,
132
      g_param_spec_int ("config-interval",
133 134
          "SPS PPS Send Interval",
          "Send SPS and PPS Insertion Interval in seconds (sprop parameter sets "
135 136 137
          "will be multiplexed in the data stream when detected.) "
          "(0 = disabled, -1 = send with every IDR frame)",
          -1, 3600, DEFAULT_CONFIG_INTERVAL,
138 139 140 141 142
          G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));

  /* Override BaseParse vfuncs */
  parse_class->start = GST_DEBUG_FUNCPTR (gst_h264_parse_start);
  parse_class->stop = GST_DEBUG_FUNCPTR (gst_h264_parse_stop);
143
  parse_class->handle_frame = GST_DEBUG_FUNCPTR (gst_h264_parse_handle_frame);
144 145 146
  parse_class->pre_push_frame =
      GST_DEBUG_FUNCPTR (gst_h264_parse_pre_push_frame);
  parse_class->set_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_set_caps);
147
  parse_class->get_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_get_caps);
148
  parse_class->sink_event = GST_DEBUG_FUNCPTR (gst_h264_parse_event);
149
  parse_class->src_event = GST_DEBUG_FUNCPTR (gst_h264_parse_src_event);
150

151 152
  gst_element_class_add_static_pad_template (gstelement_class, &srctemplate);
  gst_element_class_add_static_pad_template (gstelement_class, &sinktemplate);
153

154
  gst_element_class_set_static_metadata (gstelement_class, "H.264 parser",
155 156 157
      "Codec/Parser/Converter/Video",
      "Parses H.264 streams",
      "Mark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>");
158 159 160
}

static void
161
gst_h264_parse_init (GstH264Parse * h264parse)
162 163
{
  h264parse->frame_out = gst_adapter_new ();
164
  gst_base_parse_set_pts_interpolation (GST_BASE_PARSE (h264parse), FALSE);
165
  GST_PAD_SET_ACCEPT_INTERSECT (GST_BASE_PARSE_SINK_PAD (h264parse));
166
  GST_PAD_SET_ACCEPT_TEMPLATE (GST_BASE_PARSE_SINK_PAD (h264parse));
167 168 169 170 171 172 173 174 175
}


static void
gst_h264_parse_finalize (GObject * object)
{
  GstH264Parse *h264parse = GST_H264_PARSE (object);

  g_object_unref (h264parse->frame_out);
176 177

  G_OBJECT_CLASS (parent_class)->finalize (object);
178 179 180 181 182
}

static void
gst_h264_parse_reset_frame (GstH264Parse * h264parse)
{
183 184
  GST_DEBUG_OBJECT (h264parse, "reset frame");

185
  /* done parsing; reset state */
186
  h264parse->current_off = -1;
187

188 189 190
  h264parse->picture_start = FALSE;
  h264parse->update_caps = FALSE;
  h264parse->idr_pos = -1;
191
  h264parse->sei_pos = -1;
192
  h264parse->keyframe = FALSE;
193
  h264parse->header = FALSE;
194
  h264parse->frame_start = FALSE;
195
  gst_adapter_clear (h264parse->frame_out);
196 197 198
}

static void
199
gst_h264_parse_reset_stream_info (GstH264Parse * h264parse)
200
{
201 202
  gint i;

203 204 205 206
  h264parse->width = 0;
  h264parse->height = 0;
  h264parse->fps_num = 0;
  h264parse->fps_den = 0;
207 208
  h264parse->upstream_par_n = -1;
  h264parse->upstream_par_d = -1;
209 210
  h264parse->parsed_par_n = 0;
  h264parse->parsed_par_d = 0;
211 212
  h264parse->have_pps = FALSE;
  h264parse->have_sps = FALSE;
213

214 215 216 217
  h264parse->multiview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
  h264parse->multiview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
  h264parse->first_in_bundle = TRUE;

218 219 220
  h264parse->align = GST_H264_PARSE_ALIGN_NONE;
  h264parse->format = GST_H264_PARSE_FORMAT_NONE;

221 222 223
  h264parse->transform = FALSE;
  h264parse->nal_length_size = 4;
  h264parse->packetized = FALSE;
224
  h264parse->push_codec = FALSE;
225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240

  gst_buffer_replace (&h264parse->codec_data, NULL);
  gst_buffer_replace (&h264parse->codec_data_in, NULL);

  gst_h264_parse_reset_frame (h264parse);

  for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++)
    gst_buffer_replace (&h264parse->sps_nals[i], NULL);
  for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++)
    gst_buffer_replace (&h264parse->pps_nals[i], NULL);
}

static void
gst_h264_parse_reset (GstH264Parse * h264parse)
{
  h264parse->last_report = GST_CLOCK_TIME_NONE;
241

242 243 244 245
  h264parse->dts = GST_CLOCK_TIME_NONE;
  h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
  h264parse->do_ts = TRUE;

246 247
  h264parse->sent_codec_tag = FALSE;

248
  h264parse->pending_key_unit_ts = GST_CLOCK_TIME_NONE;
249
  gst_event_replace (&h264parse->force_key_unit_event, NULL);
250

251 252
  h264parse->discont = FALSE;

253
  gst_h264_parse_reset_stream_info (h264parse);
254 255 256 257 258 259 260
}

static gboolean
gst_h264_parse_start (GstBaseParse * parse)
{
  GstH264Parse *h264parse = GST_H264_PARSE (parse);

261
  GST_DEBUG_OBJECT (parse, "start");
262 263
  gst_h264_parse_reset (h264parse);

264 265 266 267 268 269 270
  h264parse->nalparser = gst_h264_nal_parser_new ();

  h264parse->dts = GST_CLOCK_TIME_NONE;
  h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
  h264parse->sei_pic_struct_pres_flag = FALSE;
  h264parse->sei_pic_struct = 0;
  h264parse->field_pic_flag = 0;
271

272
  gst_base_parse_set_min_frame_size (parse, 6);
273 274 275 276 277 278 279 280 281

  return TRUE;
}

static gboolean
gst_h264_parse_stop (GstBaseParse * parse)
{
  GstH264Parse *h264parse = GST_H264_PARSE (parse);

282
  GST_DEBUG_OBJECT (parse, "stop");
283 284
  gst_h264_parse_reset (h264parse);

285
  gst_h264_nal_parser_free (h264parse->nalparser);
286 287 288 289 290 291 292 293 294 295 296 297 298

  return TRUE;
}

static const gchar *
gst_h264_parse_get_string (GstH264Parse * parse, gboolean format, gint code)
{
  if (format) {
    switch (code) {
      case GST_H264_PARSE_FORMAT_AVC:
        return "avc";
      case GST_H264_PARSE_FORMAT_BYTE:
        return "byte-stream";
299 300
      case GST_H264_PARSE_FORMAT_AVC3:
        return "avc3";
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316
      default:
        return "none";
    }
  } else {
    switch (code) {
      case GST_H264_PARSE_ALIGN_NAL:
        return "nal";
      case GST_H264_PARSE_ALIGN_AU:
        return "au";
      default:
        return "none";
    }
  }
}

static void
317
gst_h264_parse_format_from_caps (GstCaps * caps, guint * format, guint * align)
318
{
319 320 321 322 323
  if (format)
    *format = GST_H264_PARSE_FORMAT_NONE;

  if (align)
    *align = GST_H264_PARSE_ALIGN_NONE;
324

325 326 327 328
  g_return_if_fail (gst_caps_is_fixed (caps));

  GST_DEBUG ("parsing caps: %" GST_PTR_FORMAT, caps);

329 330 331 332
  if (caps && gst_caps_get_size (caps) > 0) {
    GstStructure *s = gst_caps_get_structure (caps, 0);
    const gchar *str = NULL;

333 334 335 336 337 338
    if (format) {
      if ((str = gst_structure_get_string (s, "stream-format"))) {
        if (strcmp (str, "avc") == 0)
          *format = GST_H264_PARSE_FORMAT_AVC;
        else if (strcmp (str, "byte-stream") == 0)
          *format = GST_H264_PARSE_FORMAT_BYTE;
339 340
        else if (strcmp (str, "avc3") == 0)
          *format = GST_H264_PARSE_FORMAT_AVC3;
341 342 343
      }
    }

344 345 346 347 348 349
    if (align) {
      if ((str = gst_structure_get_string (s, "alignment"))) {
        if (strcmp (str, "au") == 0)
          *align = GST_H264_PARSE_ALIGN_AU;
        else if (strcmp (str, "nal") == 0)
          *align = GST_H264_PARSE_ALIGN_NAL;
350 351 352
      }
    }
  }
353 354 355 356
}

/* check downstream caps to configure format and alignment */
static void
357 358
gst_h264_parse_negotiate (GstH264Parse * h264parse, gint in_format,
    GstCaps * in_caps)
359 360
{
  GstCaps *caps;
361 362
  guint format = h264parse->format;
  guint align = h264parse->align;
363

364 365
  g_return_if_fail ((in_caps == NULL) || gst_caps_is_fixed (in_caps));

366 367 368
  caps = gst_pad_get_allowed_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
  GST_DEBUG_OBJECT (h264parse, "allowed caps: %" GST_PTR_FORMAT, caps);

369
  /* concentrate on leading structure, since decodebin parser
370 371
   * capsfilter always includes parser template caps */
  if (caps) {
Wim Taymans's avatar
Wim Taymans committed
372
    caps = gst_caps_truncate (caps);
373 374 375 376
    GST_DEBUG_OBJECT (h264parse, "negotiating with caps: %" GST_PTR_FORMAT,
        caps);
  }

377 378
  h264parse->can_passthrough = FALSE;

379 380 381 382 383 384
  if (in_caps && caps) {
    if (gst_caps_can_intersect (in_caps, caps)) {
      GST_DEBUG_OBJECT (h264parse, "downstream accepts upstream caps");
      gst_h264_parse_format_from_caps (in_caps, &format, &align);
      gst_caps_unref (caps);
      caps = NULL;
385
      h264parse->can_passthrough = TRUE;
386 387
    }
  }
388

389 390
  /* FIXME We could fail the negotiation immediatly if caps are empty */
  if (caps && !gst_caps_is_empty (caps)) {
391
    /* fixate to avoid ambiguity with lists when parsing */
Wim Taymans's avatar
Wim Taymans committed
392
    caps = gst_caps_fixate (caps);
393
    gst_h264_parse_format_from_caps (caps, &format, &align);
394
    gst_caps_unref (caps);
395
  }
396 397 398 399 400 401 402 403 404 405 406 407 408

  /* default */
  if (!format)
    format = GST_H264_PARSE_FORMAT_BYTE;
  if (!align)
    align = GST_H264_PARSE_ALIGN_AU;

  GST_DEBUG_OBJECT (h264parse, "selected format %s, alignment %s",
      gst_h264_parse_get_string (h264parse, TRUE, format),
      gst_h264_parse_get_string (h264parse, FALSE, align));

  h264parse->format = format;
  h264parse->align = align;
409

410 411
  h264parse->transform = in_format != h264parse->format ||
      align == GST_H264_PARSE_ALIGN_AU;
412 413 414 415 416 417 418
}

static GstBuffer *
gst_h264_parse_wrap_nal (GstH264Parse * h264parse, guint format, guint8 * data,
    guint size)
{
  GstBuffer *buf;
419
  guint nl = h264parse->nal_length_size;
420
  guint32 tmp;
421

422
  GST_DEBUG_OBJECT (h264parse, "nal length %d", size);
423

424
  buf = gst_buffer_new_allocate (NULL, 4 + size, NULL);
425 426
  if (format == GST_H264_PARSE_FORMAT_AVC
      || format == GST_H264_PARSE_FORMAT_AVC3) {
427
    tmp = GUINT32_TO_BE (size << (32 - 8 * nl));
428
  } else {
429 430 431 432
    /* HACK: nl should always be 4 here, otherwise this won't work. 
     * There are legit cases where nl in avc stream is 2, but byte-stream
     * SC is still always 4 bytes. */
    nl = 4;
433
    tmp = GUINT32_TO_BE (1);
434 435
  }

436 437
  gst_buffer_fill (buf, 0, &tmp, sizeof (guint32));
  gst_buffer_fill (buf, nl, data, size);
438
  gst_buffer_set_size (buf, size + nl);
439 440 441 442

  return buf;
}

443 444 445 446 447 448 449
static void
gst_h264_parser_store_nal (GstH264Parse * h264parse, guint id,
    GstH264NalUnitType naltype, GstH264NalUnit * nalu)
{
  GstBuffer *buf, **store;
  guint size = nalu->size, store_size;

450
  if (naltype == GST_H264_NAL_SPS || naltype == GST_H264_NAL_SUBSET_SPS) {
451 452
    store_size = GST_H264_MAX_SPS_COUNT;
    store = h264parse->sps_nals;
453
    GST_DEBUG_OBJECT (h264parse, "storing sps %u", id);
454 455 456
  } else if (naltype == GST_H264_NAL_PPS) {
    store_size = GST_H264_MAX_PPS_COUNT;
    store = h264parse->pps_nals;
457
    GST_DEBUG_OBJECT (h264parse, "storing pps %u", id);
458 459 460 461 462 463 464 465
  } else
    return;

  if (id >= store_size) {
    GST_DEBUG_OBJECT (h264parse, "unable to store nal, id out-of-range %d", id);
    return;
  }

466
  buf = gst_buffer_new_allocate (NULL, size, NULL);
467
  gst_buffer_fill (buf, 0, nalu->data + nalu->offset, size);
468

469 470 471 472
  /* Indicate that buffer contain a header needed for decoding */
  if (naltype == GST_H264_NAL_SPS || naltype == GST_H264_NAL_PPS)
    GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_HEADER);

473 474 475 476 477 478
  if (store[id])
    gst_buffer_unref (store[id]);

  store[id] = buf;
}

479
#ifndef GST_DISABLE_GST_DEBUG
480 481 482 483 484 485 486 487 488 489 490 491 492
static const gchar *nal_names[] = {
  "Unknown",
  "Slice",
  "Slice DPA",
  "Slice DPB",
  "Slice DPC",
  "Slice IDR",
  "SEI",
  "SPS",
  "PPS",
  "AU delimiter",
  "Sequence End",
  "Stream End",
493 494 495
  "Filler Data",
  "SPS extension",
  "Prefix",
496 497 498 499 500 501
  "SPS Subset",
  "Depth Parameter Set",
  "Reserved", "Reserved",
  "Slice Aux Unpartitioned",
  "Slice Extension",
  "Slice Depth/3D-AVC Extension"
502 503 504 505 506
};

static const gchar *
_nal_name (GstH264NalUnitType nal_type)
{
507
  if (nal_type <= GST_H264_NAL_SLICE_DEPTH)
508 509 510
    return nal_names[nal_type];
  return "Invalid";
}
511
#endif
512

513 514 515 516 517 518 519 520 521 522 523
static void
gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
{
  GstH264SEIMessage sei;
  GstH264NalParser *nalparser = h264parse->nalparser;
  GstH264ParserResult pres;
  GArray *messages;
  guint i;

  pres = gst_h264_parser_parse_sei (nalparser, nalu, &messages);
  if (pres != GST_H264_PARSER_OK)
524
    GST_WARNING_OBJECT (h264parse, "failed to parse one or more SEI message");
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551

  /* Even if pres != GST_H264_PARSER_OK, some message could have been parsed and
   * stored in messages.
   */
  for (i = 0; i < messages->len; i++) {
    sei = g_array_index (messages, GstH264SEIMessage, i);
    switch (sei.payloadType) {
      case GST_H264_SEI_PIC_TIMING:
        h264parse->sei_pic_struct_pres_flag =
            sei.payload.pic_timing.pic_struct_present_flag;
        h264parse->sei_cpb_removal_delay =
            sei.payload.pic_timing.cpb_removal_delay;
        if (h264parse->sei_pic_struct_pres_flag)
          h264parse->sei_pic_struct = sei.payload.pic_timing.pic_struct;
        GST_LOG_OBJECT (h264parse, "pic timing updated");
        break;
      case GST_H264_SEI_BUF_PERIOD:
        if (h264parse->ts_trn_nb == GST_CLOCK_TIME_NONE ||
            h264parse->dts == GST_CLOCK_TIME_NONE)
          h264parse->ts_trn_nb = 0;
        else
          h264parse->ts_trn_nb = h264parse->dts;

        GST_LOG_OBJECT (h264parse,
            "new buffering period; ts_trn_nb updated: %" GST_TIME_FORMAT,
            GST_TIME_ARGS (h264parse->ts_trn_nb));
        break;
552 553 554 555 556 557 558 559 560 561

        /* Additional messages that are not innerly useful to the
         * element but for debugging purposes */
      case GST_H264_SEI_RECOVERY_POINT:
        GST_LOG_OBJECT (h264parse, "recovery point found: %u %u %u %u",
            sei.payload.recovery_point.recovery_frame_cnt,
            sei.payload.recovery_point.exact_match_flag,
            sei.payload.recovery_point.broken_link_flag,
            sei.payload.recovery_point.changing_slice_group_idc);
        break;
562 563 564

        /* Additional messages that are not innerly useful to the
         * element but for debugging purposes */
565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
      case GST_H264_SEI_STEREO_VIDEO_INFO:{
        GstVideoMultiviewMode mview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
        GstVideoMultiviewFlags mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;

        GST_LOG_OBJECT (h264parse, "Stereo video information %u %u %u %u %u %u",
            sei.payload.stereo_video_info.field_views_flag,
            sei.payload.stereo_video_info.top_field_is_left_view_flag,
            sei.payload.stereo_video_info.current_frame_is_left_view_flag,
            sei.payload.stereo_video_info.next_frame_is_second_view_flag,
            sei.payload.stereo_video_info.left_view_self_contained_flag,
            sei.payload.stereo_video_info.right_view_self_contained_flag);

        if (sei.payload.stereo_video_info.field_views_flag) {
          mview_mode = GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
          if (!sei.payload.stereo_video_info.top_field_is_left_view_flag)
            mview_mode |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
        } else {
          mview_mode = GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
          if (sei.payload.stereo_video_info.next_frame_is_second_view_flag) {
            /* Mark current frame as first in bundle */
            h264parse->first_in_bundle = TRUE;
            if (!sei.payload.stereo_video_info.current_frame_is_left_view_flag)
              mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
          }
        }
        if (mview_mode != h264parse->multiview_mode ||
            mview_flags != h264parse->multiview_flags) {
          h264parse->multiview_mode = mview_mode;
          h264parse->multiview_flags = mview_flags;
          /* output caps need to be changed */
          gst_h264_parse_update_src_caps (h264parse, NULL);
        }
597
        break;
598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
      }
      case GST_H264_SEI_FRAME_PACKING:{
        GstVideoMultiviewMode mview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
        GstVideoMultiviewFlags mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;

        GST_LOG_OBJECT (h264parse,
            "frame packing arrangement message: id %u cancelled %u "
            "type %u quincunx %u content_interpretation %d flip %u "
            "right_first %u field_views %u is_frame0 %u",
            sei.payload.frame_packing.frame_packing_id,
            sei.payload.frame_packing.frame_packing_cancel_flag,
            sei.payload.frame_packing.frame_packing_type,
            sei.payload.frame_packing.quincunx_sampling_flag,
            sei.payload.frame_packing.content_interpretation_type,
            sei.payload.frame_packing.spatial_flipping_flag,
            sei.payload.frame_packing.frame0_flipped_flag,
            sei.payload.frame_packing.field_views_flag,
            sei.payload.frame_packing.current_frame_is_frame0_flag);

        /* Only IDs from 0->255 and 512->2^31-1 are valid. Ignore others */
        if ((sei.payload.frame_packing.frame_packing_id >= 256 &&
                sei.payload.frame_packing.frame_packing_id < 512) ||
            (sei.payload.frame_packing.frame_packing_id >= (1U << 31)))
          break;                /* ignore */

        if (!sei.payload.frame_packing.frame_packing_cancel_flag) {
          /* Cancel flag sets things back to no-info */

          if (sei.payload.frame_packing.content_interpretation_type == 2)
            mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;

          switch (sei.payload.frame_packing.frame_packing_type) {
            case 0:
              mview_mode = GST_VIDEO_MULTIVIEW_MODE_CHECKERBOARD;
              break;
            case 1:
              mview_mode = GST_VIDEO_MULTIVIEW_MODE_COLUMN_INTERLEAVED;
              break;
            case 2:
              mview_mode = GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
              break;
            case 3:
              if (sei.payload.frame_packing.quincunx_sampling_flag)
                mview_mode = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX;
              else
                mview_mode = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE;
              if (sei.payload.frame_packing.spatial_flipping_flag) {
                /* One of the views is flopped. */
                if (sei.payload.frame_packing.frame0_flipped_flag !=
                    ! !(mview_flags &
                        GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST))
                  /* the left view is flopped */
                  mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED;
                else
                  mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED;
              }
              break;
            case 4:
              mview_mode = GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM;
              if (sei.payload.frame_packing.spatial_flipping_flag) {
                /* One of the views is flipped, */
                if (sei.payload.frame_packing.frame0_flipped_flag !=
                    ! !(mview_flags &
                        GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST))
                  /* the left view is flipped */
                  mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED;
                else
                  mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED;
              }
              break;
            case 5:
              if (sei.payload.frame_packing.content_interpretation_type == 0)
                mview_mode = GST_VIDEO_MULTIVIEW_MODE_MULTIVIEW_FRAME_BY_FRAME;
              else
                mview_mode = GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
              break;
            default:
              GST_DEBUG_OBJECT (h264parse, "Invalid frame packing type %u",
                  sei.payload.frame_packing.frame_packing_type);
              break;
          }
        }

        if (mview_mode != h264parse->multiview_mode ||
            mview_flags != h264parse->multiview_flags) {
          h264parse->multiview_mode = mview_mode;
          h264parse->multiview_flags = mview_flags;
          /* output caps need to be changed */
          gst_h264_parse_update_src_caps (h264parse, NULL);
        }
688
        break;
689
      }
690 691 692 693 694
    }
  }
  g_array_free (messages, TRUE);
}

695
/* caller guarantees 2 bytes of nal payload */
696
static gboolean
697
gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
698 699
{
  guint nal_type;
700 701
  GstH264PPS pps = { 0, };
  GstH264SPS sps = { 0, };
702
  GstH264NalParser *nalparser = h264parse->nalparser;
703
  GstH264ParserResult pres;
704

705
  /* nothing to do for broken input */
706 707
  if (G_UNLIKELY (nalu->size < 2)) {
    GST_DEBUG_OBJECT (h264parse, "not processing nal size %u", nalu->size);
708
    return TRUE;
709 710
  }

711
  /* we have a peek as well */
712
  nal_type = nalu->type;
713

714 715
  GST_DEBUG_OBJECT (h264parse, "processing nal of type %u %s, size %u",
      nal_type, _nal_name (nal_type), nalu->size);
716

717
  switch (nal_type) {
718 719 720
    case GST_H264_NAL_SUBSET_SPS:
      if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
        return FALSE;
721
      pres = gst_h264_parser_parse_subset_sps (nalparser, nalu, &sps, TRUE);
722 723
      goto process_sps;

724
    case GST_H264_NAL_SPS:
725 726
      /* reset state, everything else is obsolete */
      h264parse->state = 0;
727
      pres = gst_h264_parser_parse_sps (nalparser, nalu, &sps, TRUE);
728

729
    process_sps:
730
      /* arranged for a fallback sps.id, so use that one and only warn */
731
      if (pres != GST_H264_PARSER_OK) {
732
        GST_WARNING_OBJECT (h264parse, "failed to parse SPS:");
733 734
        return FALSE;
      }
735 736 737

      GST_DEBUG_OBJECT (h264parse, "triggering src caps check");
      h264parse->update_caps = TRUE;
738 739 740 741 742 743 744 745 746
      h264parse->have_sps = TRUE;
      if (h264parse->push_codec && h264parse->have_pps) {
        /* SPS and PPS found in stream before the first pre_push_frame, no need
         * to forcibly push at start */
        GST_INFO_OBJECT (h264parse, "have SPS/PPS in stream");
        h264parse->push_codec = FALSE;
        h264parse->have_sps = FALSE;
        h264parse->have_pps = FALSE;
      }
747 748

      gst_h264_parser_store_nal (h264parse, sps.id, nal_type, nalu);
749
      gst_h264_sps_clear (&sps);
750
      h264parse->state |= GST_H264_PARSE_STATE_GOT_SPS;
751
      h264parse->header |= TRUE;
752 753
      break;
    case GST_H264_NAL_PPS:
754
      /* expected state: got-sps */
755
      h264parse->state &= GST_H264_PARSE_STATE_GOT_SPS;
756 757
      if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
        return FALSE;
758

759 760
      pres = gst_h264_parser_parse_pps (nalparser, nalu, &pps);
      /* arranged for a fallback pps.id, so use that one and only warn */
761
      if (pres != GST_H264_PARSER_OK) {
762
        GST_WARNING_OBJECT (h264parse, "failed to parse PPS:");
763 764 765
        if (pres != GST_H264_PARSER_BROKEN_LINK)
          return FALSE;
      }
766

767
      /* parameters might have changed, force caps check */
768 769 770 771
      if (!h264parse->have_pps) {
        GST_DEBUG_OBJECT (h264parse, "triggering src caps check");
        h264parse->update_caps = TRUE;
      }
772 773 774 775 776 777 778 779 780
      h264parse->have_pps = TRUE;
      if (h264parse->push_codec && h264parse->have_sps) {
        /* SPS and PPS found in stream before the first pre_push_frame, no need
         * to forcibly push at start */
        GST_INFO_OBJECT (h264parse, "have SPS/PPS in stream");
        h264parse->push_codec = FALSE;
        h264parse->have_sps = FALSE;
        h264parse->have_pps = FALSE;
      }
781 782

      gst_h264_parser_store_nal (h264parse, pps.id, nal_type, nalu);
783
      gst_h264_pps_clear (&pps);
784
      h264parse->state |= GST_H264_PARSE_STATE_GOT_PPS;
785
      h264parse->header |= TRUE;
786
      break;
787
    case GST_H264_NAL_SEI:
788 789 790 791
      /* expected state: got-sps */
      if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
        return FALSE;

792
      h264parse->header |= TRUE;
793
      gst_h264_parse_process_sei (h264parse, nalu);
794 795
      /* mark SEI pos */
      if (h264parse->sei_pos == -1) {
796
        if (h264parse->transform)
797 798
          h264parse->sei_pos = gst_adapter_available (h264parse->frame_out);
        else
799
          h264parse->sei_pos = nalu->sc_offset;
800 801 802
        GST_DEBUG_OBJECT (h264parse, "marking SEI in frame at offset %d",
            h264parse->sei_pos);
      }
803 804 805 806 807 808
      break;

    case GST_H264_NAL_SLICE:
    case GST_H264_NAL_SLICE_DPA:
    case GST_H264_NAL_SLICE_DPB:
    case GST_H264_NAL_SLICE_DPC:
809
    case GST_H264_NAL_SLICE_IDR:
810
    case GST_H264_NAL_SLICE_EXT:
811
      /* expected state: got-sps|got-pps (valid picture headers) */
812
      h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
813 814 815
      if (!GST_H264_PARSE_STATE_VALID (h264parse,
              GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS))
        return FALSE;
816

817
      /* don't need to parse the whole slice (header) here */
818
      if (*(nalu->data + nalu->offset + nalu->header_bytes) & 0x80) {
819 820 821 822
        /* means first_mb_in_slice == 0 */
        /* real frame data */
        GST_DEBUG_OBJECT (h264parse, "first_mb_in_slice = 0");
        h264parse->frame_start = TRUE;
823
      }
824
      GST_DEBUG_OBJECT (h264parse, "frame start: %i", h264parse->frame_start);
825 826
      if (nal_type == GST_H264_NAL_SLICE_EXT && !GST_H264_IS_MVC_NALU (nalu))
        break;
827 828
      {
        GstH264SliceHdr slice;
829

830 831 832 833 834
        pres = gst_h264_parser_parse_slice_hdr (nalparser, nalu, &slice,
            FALSE, FALSE);
        GST_DEBUG_OBJECT (h264parse,
            "parse result %d, first MB: %u, slice type: %u",
            pres, slice.first_mb_in_slice, slice.type);
835
        if (pres == GST_H264_PARSER_OK) {
836 837
          if (GST_H264_IS_I_SLICE (&slice) || GST_H264_IS_SI_SLICE (&slice))
            h264parse->keyframe |= TRUE;
838

839
          h264parse->state |= GST_H264_PARSE_STATE_GOT_SLICE;
840
          h264parse->field_pic_flag = slice.field_pic_flag;
841
        }
842 843 844 845
      }
      if (G_LIKELY (nal_type != GST_H264_NAL_SLICE_IDR &&
              !h264parse->push_codec))
        break;
846 847 848 849 850
      /* if we need to sneak codec NALs into the stream,
       * this is a good place, so fake it as IDR
       * (which should be at start anyway) */
      /* mark where config needs to go if interval expired */
      /* mind replacement buffer if applicable */
851
      if (h264parse->idr_pos == -1) {
852
        if (h264parse->transform)
853 854
          h264parse->idr_pos = gst_adapter_available (h264parse->frame_out);
        else
855
          h264parse->idr_pos = nalu->sc_offset;
856 857 858
        GST_DEBUG_OBJECT (h264parse, "marking IDR in frame at offset %d",
            h264parse->idr_pos);
      }
859 860 861 862 863 864
      /* if SEI preceeds (faked) IDR, then we have to insert config there */
      if (h264parse->sei_pos >= 0 && h264parse->idr_pos > h264parse->sei_pos) {
        h264parse->idr_pos = h264parse->sei_pos;
        GST_DEBUG_OBJECT (h264parse, "moved IDR mark to SEI position %d",
            h264parse->idr_pos);
      }
865
      break;
866 867 868 869 870 871
    case GST_H264_NAL_AU_DELIMITER:
      /* Just accumulate AU Delimiter, whether it's before SPS or not */
      pres = gst_h264_parser_parse_nal (nalparser, nalu);
      if (pres != GST_H264_PARSER_OK)
        return FALSE;
      break;
872
    default:
873 874 875 876 877 878 879 880
      /* drop anything before the initial SPS */
      if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
        return FALSE;

      pres = gst_h264_parser_parse_nal (nalparser, nalu);
      if (pres != GST_H264_PARSER_OK)
        return FALSE;
      break;
881 882 883 884
  }

  /* if AVC output needed, collect properly prefixed nal in adapter,
   * and use that to replace outgoing buffer data later on */
885
  if (h264parse->transform) {
886 887 888 889
    GstBuffer *buf;

    GST_LOG_OBJECT (h264parse, "collecting NAL in AVC frame");
    buf = gst_h264_parse_wrap_nal (h264parse, h264parse->format,
890
        nalu->data + nalu->offset, nalu->size);
891 892
    gst_adapter_push (h264parse->frame_out, buf);
  }
893
  return TRUE;
894 895 896 897 898
}

/* caller guarantees at least 2 bytes of nal payload for each nal
 * returns TRUE if next_nal indicates that nal terminates an AU */
static inline gboolean
899 900
gst_h264_parse_collect_nal (GstH264Parse * h264parse, const guint8 * data,
    guint size, GstH264NalUnit * nalu)
901 902
{
  gboolean complete;
903 904 905
  GstH264ParserResult parse_res;
  GstH264NalUnitType nal_type = nalu->type;
  GstH264NalUnit nnalu;
906

907
  GST_DEBUG_OBJECT (h264parse, "parsing collected nal");
908 909
  parse_res = gst_h264_parser_identify_nalu_unchecked (h264parse->nalparser,
      data, nalu->offset + nalu->size, size, &nnalu);
910

911
  if (parse_res != GST_H264_PARSER_OK)
912 913
    return FALSE;

914
  /* determine if AU complete */
915
  GST_LOG_OBJECT (h264parse, "nal type: %d %s", nal_type, _nal_name (nal_type));
916 917
  /* coded slice NAL starts a picture,
   * i.e. other types become aggregated in front of it */
918 919
  h264parse->picture_start |= (nal_type == GST_H264_NAL_SLICE ||
      nal_type == GST_H264_NAL_SLICE_DPA || nal_type == GST_H264_NAL_SLICE_IDR);
920 921 922 923 924 925 926 927

  /* consider a coded slices (IDR or not) to start a picture,
   * (so ending the previous one) if first_mb_in_slice == 0
   * (non-0 is part of previous one) */
  /* NOTE this is not entirely according to Access Unit specs in 7.4.1.2.4,
   * but in practice it works in sane cases, needs not much parsing,
   * and also works with broken frame_num in NAL
   * (where spec-wise would fail) */
928
  nal_type = nnalu.type;
929 930 931
  complete = h264parse->picture_start && ((nal_type >= GST_H264_NAL_SEI &&
          nal_type <= GST_H264_NAL_AU_DELIMITER) ||
      (nal_type >= 14 && nal_type <= 18));
932

933 934 935 936 937
  GST_LOG_OBJECT (h264parse, "next nal type: %d %s", nal_type,
      _nal_name (nal_type));
  complete |= h264parse->picture_start && (nal_type == GST_H264_NAL_SLICE
      || nal_type == GST_H264_NAL_SLICE_DPA
      || nal_type == GST_H264_NAL_SLICE_IDR) &&
938
      /* first_mb_in_slice == 0 considered start of frame */
939
      (nnalu.data[nnalu.offset + nnalu.header_bytes] & 0x80);
940 941 942 943 944 945

  GST_LOG_OBJECT (h264parse, "au complete: %d", complete);

  return complete;
}

946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986

static GstFlowReturn
gst_h264_parse_handle_frame_packetized (GstBaseParse * parse,
    GstBaseParseFrame * frame)
{
  GstH264Parse *h264parse = GST_H264_PARSE (parse);
  GstBuffer *buffer = frame->buffer;
  GstFlowReturn ret = GST_FLOW_OK;
  GstH264ParserResult parse_res;
  GstH264NalUnit nalu;
  const guint nl = h264parse->nal_length_size;
  GstMapInfo map;
  gint left;

  if (nl < 1 || nl > 4) {
    GST_DEBUG_OBJECT (h264parse, "insufficient data to split input");
    return GST_FLOW_NOT_NEGOTIATED;
  }

  /* need to save buffer from invalidation upon _finish_frame */
  if (h264parse->split_packetized)
    buffer = gst_buffer_copy (frame->buffer);

  gst_buffer_map (buffer, &map, GST_MAP_READ);

  left = map.size;

  GST_LOG_OBJECT (h264parse,
      "processing packet buffer of size %" G_GSIZE_FORMAT, map.size);

  parse_res = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
      map.data, 0, map.size, nl, &nalu);

  while (parse_res == GST_H264_PARSER_OK) {
    GST_DEBUG_OBJECT (h264parse, "AVC nal offset %d", nalu.offset + nalu.size);

    /* either way, have a look at it */
    gst_h264_parse_process_nal (h264parse, &nalu);

    /* dispatch per NALU if needed */
    if (h264parse->split_packetized) {
987 988 989 990 991 992 993 994 995
      GstBaseParseFrame tmp_frame;

      gst_base_parse_frame_init (&tmp_frame);
      tmp_frame.flags |= frame->flags;
      tmp_frame.offset = frame->offset;
      tmp_frame.overhead = frame->overhead;
      tmp_frame.buffer = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL,
          nalu.offset, nalu.size);

996 997 998 999
      /* note we don't need to come up with a sub-buffer, since
       * subsequent code only considers input buffer's metadata.
       * Real data is either taken from input by baseclass or
       * a replacement output buffer is provided anyway. */
1000 1001
      gst_h264_parse_parse_frame (parse, &tmp_frame);
      ret = gst_base_parse_finish_frame (parse, &tmp_frame, nl + nalu.size);
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
      left -= nl + nalu.size;
    }

    parse_res = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
        map.data, nalu.offset + nalu.size, map.size, nl, &nalu);
  }

  gst_buffer_unmap (buffer, &map);

  if (!h264parse->split_packetized) {
    gst_h264_parse_parse_frame (parse, frame);
    ret = gst_base_parse_finish_frame (parse, frame, map.size);
  } else {
    gst_buffer_unref (buffer);
    if (G_UNLIKELY (left)) {
      /* should not be happening for nice AVC */
      GST_WARNING_OBJECT (parse, "skipping leftover AVC data %d", left);
      frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP;
      ret = gst_base_parse_finish_frame (parse, frame, map.size);
    }
  }

  if (parse_res == GST_H264_PARSER_NO_NAL_END ||
      parse_res == GST_H264_PARSER_BROKEN_DATA) {

    if (h264parse->split_packetized) {
      GST_ELEMENT_ERROR (h264parse, STREAM, FAILED, (NULL),
          ("invalid AVC input data"));

      return GST_FLOW_ERROR;
    } else {
      /* do not meddle to much in this case */
      GST_DEBUG_OBJECT (h264parse, "parsing packet failed");
    }
  }

  return ret;
}

1041 1042 1043
static GstFlowReturn
gst_h264_parse_handle_frame (GstBaseParse * parse,
    GstBaseParseFrame * frame, gint * skipsize)
1044 1045 1046
{
  GstH264Parse *h264parse = GST_H264_PARSE (parse);
  GstBuffer *buffer = frame->buffer;
1047
  GstMapInfo map;
1048
  guint8 *data;
1049
  gsize size;
1050
  gint current_off = 0;
1051
  gboolean drain, nonext;
1052
  GstH264NalParser *nalparser = h264parse->nalparser;
1053
  GstH264NalUnit nalu;
1054
  GstH264ParserResult pres;
1055
  gint framesize;
1056
  GstFlowReturn ret;
1057

1058 1059 1060 1061 1062
  if (G_UNLIKELY (GST_BUFFER_FLAG_IS_SET (frame->buffer,
              GST_BUFFER_FLAG_DISCONT))) {
    h264parse->discont = TRUE;
  }

1063 1064 1065 1066
  /* delegate in packetized case, no skipping should be needed */
  if (h264parse->packetized)
    return gst_h264_parse_handle_frame_packetized (parse, frame);

1067 1068 1069
  gst_buffer_map (buffer, &map, GST_MAP_READ);
  data = map.data;
  size = map.size;
1070

1071
  /* expect at least 3 bytes startcode == sc, and 2 bytes NALU payload */
1072
  if (G_UNLIKELY (size < 5)) {
1073
    gst_buffer_unmap (buffer, &map);
1074 1075
    *skipsize = 1;
    return GST_FLOW_OK;
1076
  }
1077 1078 1079

  /* need to configure aggregation */
  if (G_UNLIKELY (h264parse->format == GST_H264_PARSE_FORMAT_NONE))
1080
    gst_h264_parse_negotiate (h264parse, GST_H264_PARSE_FORMAT_BYTE, NULL);
1081

1082
  /* avoid stale cached parsing state */
1083
  if (frame->flags & GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME) {
1084 1085 1086 1087 1088 1089
    GST_LOG_OBJECT (h264parse, "parsing new frame");
    gst_h264_parse_reset_frame (h264parse);
  } else {
    GST_LOG_OBJECT (h264parse, "resuming frame parsing");
  }

1090 1091 1092
  /* Always consume the entire input buffer when in_align == ALIGN_AU */
  drain = GST_BASE_PARSE_DRAINING (parse)
      || h264parse->in_align == GST_H264_PARSE_ALIGN_AU;
1093
  nonext = FALSE;
1094

1095
  current_off = h264parse->current_off;
1096 1097
  if (current_off < 0)
    current_off = 0;
1098
  g_assert (current_off < size);
1099
  GST_DEBUG_OBJECT (h264parse, "last parse position %d", current_off);
1100

1101 1102 1103 1104 1105 1106 1107 1108
  /* check for initial skip */
  if (h264parse->current_off == -1) {
    pres =
        gst_h264_parser_identify_nalu_unchecked (nalparser, data, current_off,
        size, &nalu);
    switch (pres) {
      case GST_H264_PARSER_OK:
        if (nalu.sc_offset > 0) {
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
          int i;
          gboolean is_filler_data = TRUE;
          /* Handle filler data */
          for (i = 0; i < nalu.sc_offset; i++) {
            if (data[i] != 0x00) {
              is_filler_data = FALSE;
              break;
            }
          }
          if (is_filler_data) {
            GST_DEBUG_OBJECT (parse, "Dropping filler data %d", nalu.sc_offset);
            frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP;
            gst_buffer_unmap (buffer, &map);
            ret = gst_base_parse_finish_frame (parse, frame, nalu.sc_offset);
            goto drop;
          }
1125 1126 1127 1128 1129 1130 1131 1132 1133
          *skipsize = nalu.sc_offset;
          goto skip;
        }
        break;
      case GST_H264_PARSER_NO_NAL:
        *skipsize = size - 3;
        goto skip;
        break;
      default:
1134 1135 1136 1137
        /* should not really occur either */
        GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
            ("Error parsing H.264 stream"), ("Invalid H.264 stream"));
        goto invalid_stream;
1138 1139
    }
  }
1140

1141
  while (TRUE) {
1142 1143 1144
    pres =
        gst_h264_parser_identify_nalu (nalparser, data, current_off, size,
        &nalu);
1145 1146

    switch (pres) {
1147
      case GST_H264_PARSER_OK:
1148 1149
        GST_DEBUG_OBJECT (h264parse, "complete nal (offset, size): (%u, %u) ",
            nalu.offset, nalu.size);
1150
        break;
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
      case GST_H264_PARSER_NO_NAL_END:
        GST_DEBUG_OBJECT (h264parse, "not a complete nal found at offset %u",
            nalu.offset);
        /* if draining, accept it as complete nal */
        if (drain) {
          nonext = TRUE;
          nalu.size = size - nalu.offset;
          GST_DEBUG_OBJECT (h264parse, "draining, accepting with size %u",
              nalu.size);
          /* if it's not too short at least */
          if (nalu.size < 2)
            goto broken;
          break;
        }
        /* otherwise need more */
        goto more;
1167
      case GST_H264_PARSER_BROKEN_LINK:
1168 1169 1170 1171
        GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
            ("Error parsing H.264 stream"),
            ("The link to structure needed for the parsing couldn't be found"));
        goto invalid_stream;
1172
      case GST_H264_PARSER_ERROR:
1173
        /* should not really occur either */
1174 1175 1176
        GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
            ("Error parsing H.264 stream"), ("Invalid H.264 stream"));
        goto invalid_stream;
1177
      case GST_H264_PARSER_NO_NAL:
1178 1179 1180
        GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
            ("Error parsing H.264 stream"), ("No H.264 NAL unit found"));
        goto invalid_stream;
1181 1182
      case GST_H264_PARSER_BROKEN_DATA:
        GST_WARNING_OBJECT (h264parse, "input stream is corrupt; "
1183 1184
            "it contains a NAL unit of length %u", nalu.size);
      broken:
1185 1186 1187
        /* broken nal at start -> arrange to skip it,
         * otherwise have it terminate current au
         * (and so it will be skipped on next frame round) */
1188
        if (current_off == 0) {
1189
          GST_DEBUG_OBJECT (h264parse, "skipping broken nal");
1190 1191
          *skipsize = nalu.offset;
          goto skip;
1192
        } else {
1193
          GST_DEBUG_OBJECT (h264parse, "terminating au");
1194
          nalu.size = 0;
1195
          nalu.offset = nalu.sc_offset;
1196 1197
          goto end;
        }
1198 1199 1200 1201
        break;
      default:
        g_assert_not_reached ();
        break;
1202 1203
    }

1204 1205
    GST_DEBUG_OBJECT (h264parse, "%p complete nal found. Off: %u, Size: %u",
        data, nalu.offset, nalu.size);
1206

1207 1208 1209
    /* simulate no next nal if none needed */
    nonext = nonext || (h264parse->align == GST_H264_PARSE_ALIGN_NAL);

1210
    if (!nonext) {
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
      if (nalu.offset + nalu.size + 4 + 2 > size) {
        GST_DEBUG_OBJECT (h264parse, "not enough data for next NALU");
        if (drain) {
          GST_DEBUG_OBJECT (h264parse, "but draining anyway");
          nonext = TRUE;
        } else {
          goto more;
        }
      }
    }

1222
    if (!gst_h264_parse_process_nal (h264parse, &nalu)) {
1223
      GST_WARNING_OBJECT (h264parse,
1224
          "broken/invalid nal Type: %d %s, Size: %u will be dropped",
1225
          nalu.type, _nal_name (nalu.type), nalu.size);
1226 1227 1228
      *skipsize = nalu.size;
      goto skip;
    }
1229

1230
    if (nonext)
1231
      break;
1232

1233
    /* if no next nal, we know it's complete here */
1234
    if (gst_h264_parse_collect_nal (h264parse, data, size, &nalu))
1235 1236 1237
      break;

    GST_DEBUG_OBJECT (h264parse, "Looking for more");
1238
    current_off = nalu.offset + nalu.size;
1239 1240
  }

1241
end:
1242
  framesize = nalu.offset + nalu.size;
1243

1244
  gst_buffer_unmap (buffer, &map);
1245 1246 1247 1248

  gst_h264_parse_parse_frame (parse, frame);

  return gst_base_parse_finish_frame (parse, frame, framesize);
1249

1250
more:
1251
  *skipsize = 0;
1252

1253
  /* Restart parsing from here next time */
1254 1255
  if (current_off > 0)
    h264parse->current_off = current_off;
1256

1257 1258
  /* Fall-through. */
out:
1259
  gst_buffer_unmap (buffer, &map);
1260
  return GST_FLOW_OK;
1261

1262 1263 1264 1265
drop:
  GST_DEBUG_OBJECT (h264parse, "Dropped data");
  return ret;

1266 1267
skip:
  GST_DEBUG_OBJECT (h264parse, "skipping %d", *skipsize);
1268 1269 1270 1271 1272 1273 1274
  /* If we are collecting access units, we need to preserve the initial
   * config headers (SPS, PPS et al.) and only reset the frame if another
   * slice NAL was received. This means that broken pictures are discarded */
  if (h264parse->align != GST_H264_PARSE_ALIGN_AU ||
      !(h264parse->state & GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS) ||
      (h264parse->state & GST_H264_PARSE_STATE_GOT_SLICE))
    gst_h264_parse_reset_frame (h264parse);
1275
  goto out;
1276 1277 1278 1279

invalid_stream:
  gst_buffer_unmap (buffer, &map);
  return GST_FLOW_ERROR;
1280 1281 1282 1283 1284 1285 1286 1287 1288 1289
}

/* byte together avc codec data based on collected pps and sps so far */
static GstBuffer *
gst_h264_parse_make_codec_data (GstH264Parse * h264parse)
{
  GstBuffer *buf, *nal;
  gint i, sps_size = 0, pps_size = 0, num_sps = 0, num_pps = 0;
  guint8 profile_idc = 0, profile_comp = 0, level_idc = 0;
  gboolean found = FALSE;
1290 1291
  GstMapInfo map;
  guint8 *data;
1292
  gint nl;
1293 1294 1295

  /* only nal payload in stored nals */

1296 1297
  for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
    if ((nal = h264parse->sps_nals[i])) {
1298
      gsize size = gst_buffer_get_size (nal);
1299 1300
      num_sps++;
      /* size bytes also count */
1301 1302 1303
      sps_size += size + 2;
      if (size >= 4) {
        guint8 tmp[3];
1304
        found = TRUE;
1305 1306 1307 1308
        gst_buffer_extract (nal, 1, tmp, 3);
        profile_idc = tmp[0];
        profile_comp = tmp[1];
        level_idc = tmp[2];
1309 1310 1311
      }
    }
  }
1312
  for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
1313
    if ((nal = h264parse->pps_nals[i])) {
1314 1315
      num_pps++;
      /* size bytes also count */
1316
      pps_size += gst_buffer_get_size (nal) + 2;
1317 1318 1319
    }
  }

1320
  /* AVC3 has SPS/PPS inside the stream, not in the codec_data */
1321 1322
  if (h264parse->format == GST_H264_PARSE_FORMAT_AVC3) {
    num_sps = sps_size = 0;
1323
    num_pps = pps_size = 0;
1324 1325
  }

1326 1327 1328
  GST_DEBUG_OBJECT (h264parse,
      "constructing codec_data: num_sps=%d, num_pps=%d", num_sps, num_pps);

1329 1330
  if (!found || (0 == num_pps
          && GST_H264_PARSE_FORMAT_AVC3 != h264parse->format))
1331 1332
    return NULL;

1333
  buf = gst_buffer_new_allocate (NULL, 5 + 1 + sps_size + 1 + pps_size, NULL);
1334 1335
  gst_buffer_map (buf, &map, GST_MAP_WRITE);
  data = map.data;
1336
  nl = h264parse->nal_length_size;
1337 1338 1339 1340 1341

  data[0] = 1;                  /* AVC Decoder Configuration Record ver. 1 */
  data[1] = profile_idc;        /* profile_idc                             */
  data[2] = profile_comp;       /* profile_compability                     */
  data[3] = level_idc;          /* level_idc                               */
1342
  data[4] = 0xfc | (nl - 1);    /* nal_length_size_minus1                  */
1343 1344 1345
  data[5] = 0xe0 | num_sps;     /* number of SPSs */

  data += 6;
1346 1347 1348 1349 1350 1351 1352 1353
  if (h264parse->format != GST_H264_PARSE_FORMAT_AVC3) {
    for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
      if ((nal = h264parse->sps_nals[i])) {
        gsize nal_size = gst_buffer_get_size (nal);
        GST_WRITE_UINT16_BE (data, nal_size);
        gst_buffer_extract (nal, 0, data + 2, nal_size);
        data += 2 + nal_size;
      }
1354 1355 1356 1357 1358
    }
  }

  data[0] = num_pps;
  data++;
1359 1360 1361 1362 1363 1364 1365 1366
  if (h264parse->format != GST_H264_PARSE_FORMAT_AVC3) {
    for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
      if ((nal = h264parse->pps_nals[i])) {
        gsize nal_size = gst_buffer_get_size (nal);
        GST_WRITE_UINT16_BE (data, nal_size);
        gst_buffer_extract (nal, 0, data + 2, nal_size);
        data += 2 + nal_size;
      }
1367 1368 1369
    }
  }

1370
  gst_buffer_unmap (buf, &map);
1371

1372 1373 1374
  return buf;
}

1375 1376 1377
static void
gst_h264_parse_get_par (GstH264Parse * h264parse, gint * num, gint * den)
{
1378 1379 1380
  if (h264parse->upstream_par_n != -1 && h264parse->upstream_par_d != -1) {
    *num = h264parse->upstream_par_n;
    *den = h264parse->upstream_par_d;
1381 1382 1383
  } else {
    *num = h264parse->parsed_par_n;
    *den = h264parse->parsed_par_d;
1384 1385 1386
  }
}

1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417
static GstCaps *
get_compatible_profile_caps (GstH264SPS * sps)
{
  GstCaps *caps = NULL;
  const gchar **profiles = NULL;
  gint i;
  GValue compat_profiles = G_VALUE_INIT;
  g_value_init (&compat_profiles, GST_TYPE_LIST);

  switch (sps->profile_idc) {
    case GST_H264_PROFILE_EXTENDED:
      if (sps->constraint_set0_flag) {  /* A.2.1 */
        if (sps->constraint_set1_flag) {
          static const gchar *profile_array[] =
              { "constrained-baseline", "baseline", "main", "high",
            "high-10", "high-4:2:2", "high-4:4:4", NULL
          };
          profiles = profile_array;
        } else {
          static const gchar *profile_array[] = { "baseline", NULL };
          profiles = profile_array;
        }
      } else if (sps->constraint_set1_flag) {   /* A.2.2 */
        static const gchar *profile_array[] =
            { "main", "high", "high-10", "high-4:2:2", "high-4:4:4", NULL };
        profiles = profile_array;
      }
      break;
    case GST_H264_PROFILE_BASELINE:
      if (sps->constraint_set1_flag) {  /* A.2.1 */
        static const gchar *profile_array[] =