ext/qt/gstqtglutility.cc

@@ -307,9 +307,12 @@ qt_opengl_native_context_from_gst_gl_context (GstGLContext * context)
 #if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11)
     if (platform == GST_GL_PLATFORM_GLX) {
         GstGLDisplay *display = gst_gl_context_get_display (context);
+        GstGLWindow *window = gst_gl_context_get_window (context);
         Display *xdisplay = (Display *) gst_gl_display_get_handle (display);
+        Window win = gst_gl_window_get_window_handle (window);
+        gst_object_unref (window);
         gst_object_unref (display);
-        return QVariant::fromValue(QGLXNativeContext((GLXContext) handle, xdisplay));
+        return QVariant::fromValue(QGLXNativeContext((GLXContext) handle, xdisplay, win));
     }
 #endif
 #if GST_GL_HAVE_PLATFORM_EGL && (defined (HAVE_QT_WAYLAND) || defined (HAVE_QT_EGLFS) || defined (HAVE_QT_ANDROID))

ext/qt/qtglrenderer.cc

@@ -104,13 +104,25 @@ void GstAnimationDriver::setNextTime(qint64 ms)
     m_next = ms;
 }
 
+typedef enum
+{
+  STATE_ERROR = -1,
+  STATE_NEW = 0,
+  STATE_WAITING_FOR_WINDOW,
+  STATE_WINDOW_CREATED,
+  STATE_READY,
+} SharedRenderDataState;
+
 struct SharedRenderData
 {
   volatile int refcount;
+  SharedRenderDataState state;
   GMutex lock;
+  GCond cond;
   GstAnimationDriver *m_animationDriver;
   QOpenGLContext *m_context;
   GstBackingSurface *m_surface;
+  QThread *m_renderThread;
 };
 
 static struct SharedRenderData *
@@ -205,18 +217,78 @@ GstQuickRenderer::GstQuickRenderer()
       gl_mem(NULL),
       m_sharedRenderData(NULL)
 {
-    init_debug ();
+  init_debug ();
 }
 
 static gpointer
 dup_shared_render_data (gpointer data, gpointer user_data)
 {
-    struct SharedRenderData *render_data = (struct SharedRenderData *) data;
+  struct SharedRenderData *render_data = (struct SharedRenderData *) data;
+
+  if (render_data)
+    return shared_render_data_ref (render_data);
+
+  return NULL;
+}
+
+class CreateSurfaceEvent : public QEvent
+{
+public:
+  CreateSurfaceEvent (CreateSurfaceWorker * worker)
+      : QEvent(CreateSurfaceEvent::type())
+  {
+    m_worker = worker;
+  }
+
+  ~CreateSurfaceEvent()
+  {
+    GST_TRACE ("%p destroying create surface event", this);
+    delete m_worker;
+  }
+
+  static QEvent::Type type()
+  {
+    if (customEventType == QEvent::None) {
+      int generatedType = QEvent::registerEventType();
+      customEventType = static_cast<QEvent::Type>(generatedType);
+    }
+    return customEventType;
+  }
+
+private:
+  static QEvent::Type customEventType;
+  CreateSurfaceWorker *m_worker;
+};
+
+QEvent::Type CreateSurfaceEvent::customEventType = QEvent::None;
+
+
+CreateSurfaceWorker::CreateSurfaceWorker (struct SharedRenderData * rdata)
+{
+  m_sharedRenderData = shared_render_data_ref (rdata);
+}
 
-    if (render_data)
-      return shared_render_data_ref (render_data);
+CreateSurfaceWorker::~CreateSurfaceWorker ()
+{
+  shared_render_data_unref (m_sharedRenderData);
+}
+
+bool CreateSurfaceWorker::event(QEvent * ev)
+{
+    if (ev->type() == CreateSurfaceEvent::type()) {
+        GST_TRACE ("%p creating surface", m_sharedRenderData);
+        /* create the window surface in the main thread */
+        g_mutex_lock (&m_sharedRenderData->lock);
+        m_sharedRenderData->m_surface = new GstBackingSurface;
+        m_sharedRenderData->m_surface->create();
+        m_sharedRenderData->m_surface->moveToThread (m_sharedRenderData->m_renderThread);
+        GST_TRACE ("%p created surface %p", m_sharedRenderData,
+            m_sharedRenderData->m_surface);
+        g_cond_broadcast (&m_sharedRenderData->cond);
+        g_mutex_unlock (&m_sharedRenderData->lock);
+    }
 
-    return NULL;
+    return QObject::event(ev);
 }
 
 bool GstQuickRenderer::init (GstGLContext * context, GError ** error)
@@ -232,7 +304,6 @@ bool GstQuickRenderer::init (GstGLContext * context, GError ** error)
             "native context");
         return false;
     }
-    QThread *renderThread = QThread::currentThread();
 
     struct SharedRenderData *render_data = NULL, *old_render_data;
     do {
@@ -249,46 +320,101 @@ bool GstQuickRenderer::init (GstGLContext * context, GError ** error)
                 "qt.gl.render.shared.data", old_render_data, render_data,
                 NULL, NULL));
     m_sharedRenderData = render_data;
+    GST_TRACE ("%p retrieved shared render data %p", this, m_sharedRenderData);
 
     g_mutex_lock (&m_sharedRenderData->lock);
-    if (!m_sharedRenderData->m_context) {
-        m_sharedRenderData->m_context = new QOpenGLContext;
-        GST_TRACE ("%p new QOpenGLContext %p", this, m_sharedRenderData->m_context);
-        m_sharedRenderData->m_context->setNativeHandle(qt_native_context);
+    if (m_sharedRenderData->state == STATE_ERROR) {
+        g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
+            "In an error state from a previous attempt");
+        g_mutex_unlock (&m_sharedRenderData->lock);
+        return false;
+    }
 
-        m_sharedRenderData->m_surface = new GstBackingSurface;
-        m_sharedRenderData->m_surface->create();  /* FIXME: may need to be called on Qt's main thread */
-
-        gst_gl_context_activate (context, FALSE);
-
-        /* Qt does some things that it may require the OpenGL context current in
-         * ->create() so that it has the necessry information to create the
-         * QOpenGLContext from the native handle. This may fail if the OpenGL
-         * context is already current in another thread so we need to deactivate
-         * the context from GStreamer's thread before asking Qt to create the
-         * QOpenGLContext with ->create().
-         */
-        m_sharedRenderData->m_context->create();
-        m_sharedRenderData->m_context->doneCurrent();
-
-        m_sharedRenderData->m_context->moveToThread (renderThread);
-        if (!m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface)) {
-            g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
-                "Could not make Qt OpenGL context current");
-            /* try to keep the same OpenGL context state */
-            gst_gl_context_activate (context, TRUE);
-            g_mutex_unlock (&m_sharedRenderData->lock);
-            return false;
+    if (m_sharedRenderData->state != STATE_READY) {
+        /* this state handling and locking is so that two qtglrenderer's will
+         * not attempt to create an OpenGL context without freeing the previous
+         * OpenGL context and cause a leak.  It also only allows one
+         * CreateSurfaceEvent() to be posted to the main thread
+         * (QCoreApplication::instance()->thread()) while still allowing
+         * multiple waiters to wait for the window to be created */
+        if (m_sharedRenderData->state == STATE_NEW) {
+            QCoreApplication *app = QCoreApplication::instance ();
+
+            if (!app) {
+                g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
+                    "Could not retrieve QCoreApplication instance");
+                m_sharedRenderData->state = STATE_ERROR;
+                g_mutex_unlock (&m_sharedRenderData->lock);
+                return false;
+            }
+
+            m_sharedRenderData->m_renderThread = QThread::currentThread();
+            m_sharedRenderData->m_context = new QOpenGLContext;
+            GST_TRACE ("%p new QOpenGLContext %p", this, m_sharedRenderData->m_context);
+            m_sharedRenderData->m_context->setNativeHandle(qt_native_context);
+
+            CreateSurfaceWorker *w = new CreateSurfaceWorker (m_sharedRenderData);
+            GST_TRACE ("%p posting create surface event to main thread with "
+                "worker %p", this, w);
+            w->moveToThread (app->thread());
+            app->postEvent (w, new CreateSurfaceEvent (w));
+            m_sharedRenderData->state = STATE_WAITING_FOR_WINDOW;
         }
 
-        if (!gst_gl_context_activate (context, TRUE)) {
-            g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
-                "Could not make OpenGL context current again");
-            g_mutex_unlock (&m_sharedRenderData->lock);
-            return false;
+        if (m_sharedRenderData->state == STATE_WAITING_FOR_WINDOW) {
+            gint64 end_time = g_get_monotonic_time () + 5 * G_TIME_SPAN_SECOND;
+            while (!m_sharedRenderData->m_surface) {
+                /* XXX: This might deadlock with the main thread if the
+                 * QCoreApplication is not running and will not be able to
+                 * execute.  We only wait for 5 seconds until a better
+                 * approach can be found here */
+                if (!g_cond_wait_until (&m_sharedRenderData->cond,
+                        &m_sharedRenderData->lock, end_time)) {
+                    g_set_error (error, GST_RESOURCE_ERROR,
+                        GST_RESOURCE_ERROR_NOT_FOUND,
+                        "Could not create Qt window within 5 seconds");
+                    m_sharedRenderData->state = STATE_ERROR;
+                    g_mutex_unlock (&m_sharedRenderData->lock);
+                    return false;
+                }
+            }
+
+            GST_TRACE ("%p surface successfully created", this);
+            m_sharedRenderData->state = STATE_WINDOW_CREATED;
+        }
+
+        if (m_sharedRenderData->state == STATE_WINDOW_CREATED) {
+            /* Qt does some things that may require the OpenGL context current
+             * in ->create() so that it has the necessry information to create
+             * the QOpenGLContext from the native handle. This may fail if the
+             * OpenGL context is already current in another thread so we need
+             * to deactivate the context from GStreamer's thread before asking
+             * Qt to create the QOpenGLContext with ->create().
+             */
+            gst_gl_context_activate (context, FALSE);
+            m_sharedRenderData->m_context->create();
+            m_sharedRenderData->m_context->doneCurrent();
+
+            if (!m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface)) {
+                g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
+                    "Could not make Qt OpenGL context current");
+                /* try to keep the same OpenGL context state */
+                gst_gl_context_activate (context, TRUE);
+                m_sharedRenderData->state = STATE_ERROR;
+                g_mutex_unlock (&m_sharedRenderData->lock);
+                return false;
+            }
+
+            if (!gst_gl_context_activate (context, TRUE)) {
+                g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
+                    "Could not make OpenGL context current again");
+                m_sharedRenderData->state = STATE_ERROR;
+                g_mutex_unlock (&m_sharedRenderData->lock);
+                return false;
+            }
+            m_sharedRenderData->state = STATE_READY;
         }
     }
-    g_mutex_unlock (&m_sharedRenderData->lock);
 
     m_renderControl = new QQuickRenderControl();
     /* Create a QQuickWindow that is associated with our render control. Note that this
@@ -297,7 +423,8 @@ bool GstQuickRenderer::init (GstGLContext * context, GError ** error)
      */
     m_quickWindow = new QQuickWindow(m_renderControl);
     /* after QQuickWindow creation as QQuickRenderControl requires it */
-    m_renderControl->prepareThread (renderThread);
+    m_renderControl->prepareThread (m_sharedRenderData->m_renderThread);
+    g_mutex_unlock (&m_sharedRenderData->lock);
 
     /* Create a QML engine. */
     m_qmlEngine = new QQmlEngine;

ext/qt/qtglrenderer.h

@@ -106,4 +106,18 @@ private:
     struct SharedRenderData *m_sharedRenderData;
 };
 
+class CreateSurfaceWorker : public QObject
+{
+  Q_OBJECT
+
+public:
+  CreateSurfaceWorker (struct SharedRenderData * rdata);
+  ~CreateSurfaceWorker ();
+
+  bool event(QEvent *ev) override;
+
+private:
+  struct SharedRenderData *m_sharedRenderData;
+};
+
 #endif /* __QT_QUICK_RENDER_H__ */

gst/flv/gstflvmux.c

@@ -54,12 +54,14 @@ enum
   PROP_0,
   PROP_STREAMABLE,
   PROP_METADATACREATOR,
-  PROP_ENCODER
+  PROP_ENCODER,
+  PROP_SKIP_BACKWARDS_STREAMS,
 };
 
 #define DEFAULT_STREAMABLE FALSE
 #define MAX_INDEX_ENTRIES 128
 #define DEFAULT_METADATACREATOR "GStreamer " PACKAGE_VERSION " FLV muxer"
+#define DEFAULT_SKIP_BACKWARDS_STREAMS FALSE
 
 static GstStaticPadTemplate src_templ = GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
@@ -138,7 +140,7 @@ gst_flv_mux_pad_flush (GstAggregatorPad * pad, GstAggregator * aggregator)
 {
   GstFlvMuxPad *flvpad = GST_FLV_MUX_PAD (pad);
 
-  flvpad->last_timestamp = 0;
+  flvpad->last_timestamp = GST_CLOCK_TIME_NONE;
   flvpad->pts = GST_CLOCK_STIME_NONE;
   flvpad->dts = GST_CLOCK_STIME_NONE;
 
@@ -236,6 +238,12 @@ gst_flv_mux_class_init (GstFlvMuxClass * klass)
       g_param_spec_string ("encoder", "encoder",
           "The value of encoder in the meta packet.",
           NULL, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+  g_object_class_install_property (gobject_class, PROP_SKIP_BACKWARDS_STREAMS,
+      g_param_spec_boolean ("skip-backwards-streams", "Skip backwards streams",
+          "If set to true, streams that go backwards related to the other stream "
+          "will have buffers dropped until they reach the correct timestamp",
+          DEFAULT_SKIP_BACKWARDS_STREAMS,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
   gstaggregator_class->create_new_pad =
       GST_DEBUG_FUNCPTR (gst_flv_mux_create_new_pad);
@@ -662,6 +670,7 @@ gst_flv_mux_reset_pad (GstFlvMuxPad * pad)
   pad->width = G_MAXUINT;
   pad->channels = G_MAXUINT;
   pad->info_changed = FALSE;
+  pad->drop_deltas = FALSE;
 
   gst_flv_mux_pad_flush (GST_AGGREGATOR_PAD_CAST (pad), NULL);
 }
@@ -677,8 +686,10 @@ gst_flv_mux_create_new_pad (GstAggregator * agg,
   const gchar *name = NULL;
   gboolean video;
 
-  if (mux->state != GST_FLV_MUX_STATE_HEADER) {
-    GST_WARNING_OBJECT (mux, "Can't request pads after writing header");
+  if (mux->state != GST_FLV_MUX_STATE_HEADER && !mux->streamable) {
+    GST_ELEMENT_WARNING (mux, STREAM, MUX,
+        ("Requested a late stream in a non-streamable file"),
+        ("Stream added after file started and therefore won't be playable"));
     return NULL;
   }
 
@@ -1129,7 +1140,6 @@ gst_flv_mux_create_metadata (GstFlvMux * mux)
   data[2] = 9;                  /* end marker */
   script_tag = gst_buffer_append (script_tag, tmp);
 
-
   _gst_buffer_new_and_alloc (4, &tmp, &data);
   GST_WRITE_UINT32_BE (data, gst_buffer_get_size (script_tag));
   script_tag = gst_buffer_append (script_tag, tmp);
@@ -1157,11 +1167,13 @@ gst_flv_mux_buffer_to_tag_internal (GstFlvMux * mux, GstBuffer * buffer,
   guint8 *data, *bdata = NULL;
   gsize bsize = 0;
 
-  if (!GST_CLOCK_STIME_IS_VALID (pad->dts)) {
-    pts = dts = pad->last_timestamp / GST_MSECOND;
-  } else {
+  if (GST_CLOCK_STIME_IS_VALID (pad->dts)) {
     pts = pad->pts / GST_MSECOND;
     dts = pad->dts / GST_MSECOND;
+  } else if (GST_CLOCK_TIME_IS_VALID (pad->last_timestamp)) {
+    pts = dts = pad->last_timestamp / GST_MSECOND;
+  } else {
+    pts = dts = mux->last_dts;
   }
 
   /* We prevent backwards timestamps because they confuse librtmp,
@@ -1176,7 +1188,6 @@ gst_flv_mux_buffer_to_tag_internal (GstFlvMux * mux, GstBuffer * buffer,
   }
   mux->last_dts = dts;
 
-
   /* Be safe in case TS are buggy */
   if (pts > dts)
     cts = pts - dts;
@@ -1230,7 +1241,6 @@ gst_flv_mux_buffer_to_tag_internal (GstFlvMux * mux, GstBuffer * buffer,
   data[2] = ((size - 11 - 4) >> 8) & 0xff;
   data[3] = ((size - 11 - 4) >> 0) & 0xff;
 
-
   GST_WRITE_UINT24_BE (data + 4, dts);
   data[7] = (((guint) dts) >> 24) & 0xff;
 
@@ -1605,7 +1615,6 @@ gst_flv_mux_write_buffer (GstFlvMux * mux, GstFlvMuxPad * pad,
   if (ret == GST_FLOW_OK && GST_CLOCK_TIME_IS_VALID (dts))
     pad->last_timestamp = dts;
 
-
   return ret;
 }
 
@@ -1667,7 +1676,6 @@ gst_flv_mux_query_upstream_duration (GstFlvMux * mux)
   return cb_data.duration;
 }
 
-
 static gboolean
 gst_flv_mux_are_all_pads_eos (GstFlvMux * mux)
 {
@@ -1855,22 +1863,52 @@ gst_flv_mux_find_best_pad (GstAggregator * aggregator, GstClockTime * ts)
     switch (gst_iterator_next (pads, &padptr)) {
       case GST_ITERATOR_OK:{
 
+        GstFlvMux *mux = GST_FLV_MUX (aggregator);
         GstAggregatorPad *apad = g_value_get_object (&padptr);
+        GstFlvMuxPad *fpad = GST_FLV_MUX_PAD (apad);
+        gint64 t = GST_CLOCK_TIME_NONE;
+
         buffer = gst_aggregator_pad_peek_buffer (apad);
         if (!buffer)
           continue;
-        if (best_ts == GST_CLOCK_TIME_NONE) {
-          gst_object_replace ((GstObject **) & best, GST_OBJECT (apad));
-          best_ts = gst_flv_mux_segment_to_running_time (&apad->segment,
-              GST_BUFFER_DTS_OR_PTS (buffer));
-        } else if (GST_BUFFER_DTS_OR_PTS (buffer) != GST_CLOCK_TIME_NONE) {
-          gint64 t = gst_flv_mux_segment_to_running_time (&apad->segment,
+
+        if (fpad->drop_deltas) {
+          if (mux->skip_backwards_streams
+              && GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)) {
+            GST_INFO_OBJECT (apad,
+                "Dropped buffer %" GST_PTR_FORMAT " until keyframe", buffer);
+            gst_buffer_unref (buffer);
+            gst_aggregator_pad_drop_buffer (apad);
+            continue;
+          } else {
+            /* drop-deltas is set and the buffer isn't delta, drop flag */
+            fpad->drop_deltas = FALSE;
+          }
+        }
+
+        if (GST_CLOCK_TIME_IS_VALID (GST_BUFFER_DTS_OR_PTS (buffer))) {
+          t = gst_flv_mux_segment_to_running_time (&apad->segment,
               GST_BUFFER_DTS_OR_PTS (buffer));
-          if (t < best_ts) {
-            gst_object_replace ((GstObject **) & best, GST_OBJECT (apad));
-            best_ts = t;
+
+          if (mux->skip_backwards_streams
+              && (t < (GST_MSECOND * mux->last_dts))) {
+            GST_WARNING_OBJECT (mux,
+                "Timestamp %" GST_TIME_FORMAT " going "
+                "backwards from last known %" GST_TIME_FORMAT ", dropping",
+                GST_TIME_ARGS (t), GST_TIME_ARGS (GST_MSECOND * mux->last_dts));
+            gst_buffer_unref (buffer);
+            gst_aggregator_pad_drop_buffer (apad);
+            /* Look for non-delta buffer */
+            fpad->drop_deltas = TRUE;
+            continue;
           }
         }
+
+        if (!GST_CLOCK_TIME_IS_VALID (best_ts) ||
+            (GST_CLOCK_TIME_IS_VALID (t) && t < best_ts)) {
+          gst_object_replace ((GstObject **) & best, GST_OBJECT (apad));
+          best_ts = t;
+        }
         gst_buffer_unref (buffer);
         g_value_reset (&padptr);
         break;
@@ -1918,12 +1956,15 @@ gst_flv_mux_aggregate (GstAggregator * aggregator, gboolean timeout)
       return GST_FLOW_ERROR;
     }
 
+    best = gst_flv_mux_find_best_pad (aggregator, &ts);
+
     ret = gst_flv_mux_write_header (mux);
-    if (ret != GST_FLOW_OK)
+    if (ret != GST_FLOW_OK) {
+      gst_object_unref (best);
       return ret;
-    mux->state = GST_FLV_MUX_STATE_DATA;
+    }
 
-    best = gst_flv_mux_find_best_pad (aggregator, &ts);
+    mux->state = GST_FLV_MUX_STATE_DATA;
 
     if (!mux->streamable || mux->first_timestamp == GST_CLOCK_STIME_NONE) {
       if (best && GST_CLOCK_STIME_IS_VALID (ts))
@@ -2010,6 +2051,9 @@ gst_flv_mux_get_property (GObject * object,
     case PROP_ENCODER:
       g_value_set_string (value, mux->encoder);
       break;
+    case PROP_SKIP_BACKWARDS_STREAMS:
+      g_value_set_boolean (value, mux->skip_backwards_streams);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -2050,6 +2094,9 @@ gst_flv_mux_set_property (GObject * object,
         mux->encoder = g_value_dup_string (value);
       }
       break;
+    case PROP_SKIP_BACKWARDS_STREAMS:
+      mux->skip_backwards_streams = g_value_get_boolean (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;

gst/flv/gstflvmux.h

@@ -70,6 +70,7 @@ struct _GstFlvMuxPad
   gint64 dts;
 
   gboolean info_changed;
+  gboolean drop_deltas;
 };
 
 struct _GstFlvMuxPadClass {
@@ -94,6 +95,7 @@ struct _GstFlvMux {
   gboolean streamable;
   gchar *metadatacreator;
   gchar *encoder;
+  gboolean skip_backwards_streams;
 
   GstTagList *tags;
   gboolean new_tags;