Commit 47121fe9 authored by Guillaume Desmottes's avatar Guillaume Desmottes 🐐 Committed by Sebastian Dröge

gstreamer-video: VideoDecoder bindings

The VideoCodecFrame and VideoCodecState is C API is unfortunatelly unsafe
by design. So we workarounded it by ensuring the decoder stream lock was
hold while user has a writable reference on those objects.

Based on previous work from Thibault Saunier and Philippe Normand.

Fixes #161
parent a986914b
Pipeline #38231 passed with stages
in 18 minutes and 8 seconds
......@@ -18,6 +18,7 @@ external_libraries = [
]
generate = [
"GstVideo.VideoCodecFrameFlags",
"GstVideo.VideoFormat",
"GstVideo.VideoFormatFlags",
"GstVideo.VideoTileMode",
......@@ -43,7 +44,16 @@ manual = [
"GObject.Object",
"Gst.Object",
"Gst.Element",
"Gst.Buffer",
"Gst.BufferPool",
"Gst.BufferPoolAcquireParams",
"Gst.ClockTimeDiff",
"Gst.FlowReturn",
"Gst.TagList",
"Gst.TagMergeMode",
"GstBase.BaseTransform",
"GstVideo.VideoCodecState",
"GstVideo.VideoCodecFrame",
"GstVideo.VideoInfo",
"GstVideo.VideoFormatInfo",
"GstVideo.VideoColorimetry",
......@@ -66,3 +76,68 @@ status = "generate"
name = "set_render_rectangle"
[object.function.return]
bool_return_is_error = "Failed to set render rectangle"
[[object]]
name = "GstVideo.VideoDecoder"
subclassing = true
status = "generate"
[[object.function]]
name = "allocate_output_frame"
ignore = true
[[object.function]]
name = "allocate_output_frame_with_params"
ignore = true
[[object.function]]
name = "finish_frame"
ignore = true
[[object.function]]
name = "release_frame"
ignore = true
[[object.function]]
name = "drop_frame"
ignore = true
[[object.function]]
name = "have_frame"
ignore = true
[[object.function]]
name = "set_latency"
ignore = true
[[object.function]]
name = "get_latency"
ignore = true
[[object.function]]
name = "get_frame"
ignore = true
[[object.function]]
name = "get_frames"
ignore = true
[[object.function]]
name = "get_oldest_frame"
ignore = true
[[object.function]]
name = "get_output_state"
ignore = true
[[object.function]]
name = "set_output_state"
ignore = true
[[object.function]]
name = "set_interlaced_output_state"
ignore = true
[[object.function]]
name = "negotiate"
ignore = true
......@@ -37,3 +37,4 @@ v1_16 = ["gstreamer-sys/v1_16", "gstreamer-video-sys/v1_16", "v1_14"]
embed-lgpl-docs = ["rustdoc-stripper"]
purge-lgpl-docs = ["rustdoc-stripper"]
dox = ["gstreamer-video-sys/dox", "glib/dox", "gstreamer/dox"]
subclassing = ["gstreamer/subclassing"]
......@@ -67,6 +67,32 @@ impl SetValue for VideoChromaSite {
}
}
bitflags! {
pub struct VideoCodecFrameFlags: u32 {
const DECODE_ONLY = 1;
const SYNC_POINT = 2;
const FORCE_KEYFRAME = 4;
const FORCE_KEYFRAME_HEADERS = 8;
}
}
#[doc(hidden)]
impl ToGlib for VideoCodecFrameFlags {
type GlibType = gst_video_sys::GstVideoCodecFrameFlags;
fn to_glib(&self) -> gst_video_sys::GstVideoCodecFrameFlags {
self.bits()
}
}
#[doc(hidden)]
impl FromGlib<gst_video_sys::GstVideoCodecFrameFlags> for VideoCodecFrameFlags {
fn from_glib(value: gst_video_sys::GstVideoCodecFrameFlags) -> VideoCodecFrameFlags {
skip_assert_initialized!();
VideoCodecFrameFlags::from_bits_truncate(value)
}
}
bitflags! {
pub struct VideoFlags: u32 {
const NONE = 0;
......
......@@ -2,6 +2,10 @@
// from gir-files (https://github.com/gtk-rs/gir-files)
// DO NOT EDIT
mod video_decoder;
pub use self::video_decoder::{VideoDecoder, VideoDecoderClass, NONE_VIDEO_DECODER};
pub use self::video_decoder::VideoDecoderExt;
mod video_filter;
pub use self::video_filter::{VideoFilter, VideoFilterClass, NONE_VIDEO_FILTER};
......@@ -25,6 +29,7 @@ pub use self::enums::VideoTransferFunction;
mod flags;
pub use self::flags::VideoChromaSite;
pub use self::flags::VideoCodecFrameFlags;
pub use self::flags::VideoFlags;
pub use self::flags::VideoFormatFlags;
pub use self::flags::VideoFrameFlags;
......@@ -35,5 +40,6 @@ pub use self::flags::VideoTimeCodeFlags;
#[doc(hidden)]
pub mod traits {
pub use super::VideoDecoderExt;
pub use super::VideoOverlayExt;
}
Generated by gir (https://github.com/gtk-rs/gir @ 58cffd4)
Generated by gir (https://github.com/gtk-rs/gir @ 6d4ca01)
from gir-files (https://github.com/gtk-rs/gir-files @ ???)
// This file was generated by gir (https://github.com/gtk-rs/gir)
// from gir-files (https://github.com/gtk-rs/gir-files)
// DO NOT EDIT
use VideoCodecFrame;
use glib::object::IsA;
use glib::translate::*;
use gst;
use gst_video_sys;
glib_wrapper! {
pub struct VideoDecoder(Object<gst_video_sys::GstVideoDecoder, gst_video_sys::GstVideoDecoderClass, VideoDecoderClass>) @extends gst::Element, gst::Object;
match fn {
get_type => || gst_video_sys::gst_video_decoder_get_type(),
}
}
unsafe impl Send for VideoDecoder {}
unsafe impl Sync for VideoDecoder {}
pub const NONE_VIDEO_DECODER: Option<&VideoDecoder> = None;
pub trait VideoDecoderExt: 'static {
fn add_to_frame(&self, n_bytes: i32);
fn allocate_output_buffer(&self) -> Option<gst::Buffer>;
//fn get_allocator(&self, allocator: /*Ignored*/gst::Allocator, params: /*Ignored*/gst::AllocationParams);
fn get_buffer_pool(&self) -> Option<gst::BufferPool>;
fn get_estimate_rate(&self) -> i32;
fn get_max_decode_time(&self, frame: &VideoCodecFrame) -> gst::ClockTimeDiff;
fn get_max_errors(&self) -> i32;
fn get_needs_format(&self) -> bool;
fn get_packetized(&self) -> bool;
fn get_pending_frame_size(&self) -> usize;
fn get_qos_proportion(&self) -> f64;
fn merge_tags(&self, tags: Option<&gst::TagList>, mode: gst::TagMergeMode);
fn proxy_getcaps(&self, caps: Option<&gst::Caps>, filter: Option<&gst::Caps>) -> Option<gst::Caps>;
fn set_estimate_rate(&self, enabled: bool);
fn set_max_errors(&self, num: i32);
fn set_needs_format(&self, enabled: bool);
fn set_packetized(&self, packetized: bool);
fn set_use_default_pad_acceptcaps(&self, use_: bool);
}
impl<O: IsA<VideoDecoder>> VideoDecoderExt for O {
fn add_to_frame(&self, n_bytes: i32) {
unsafe {
gst_video_sys::gst_video_decoder_add_to_frame(self.as_ref().to_glib_none().0, n_bytes);
}
}
fn allocate_output_buffer(&self) -> Option<gst::Buffer> {
unsafe {
from_glib_full(gst_video_sys::gst_video_decoder_allocate_output_buffer(self.as_ref().to_glib_none().0))
}
}
//fn get_allocator(&self, allocator: /*Ignored*/gst::Allocator, params: /*Ignored*/gst::AllocationParams) {
// unsafe { TODO: call gst_video_sys:gst_video_decoder_get_allocator() }
//}
fn get_buffer_pool(&self) -> Option<gst::BufferPool> {
unsafe {
from_glib_full(gst_video_sys::gst_video_decoder_get_buffer_pool(self.as_ref().to_glib_none().0))
}
}
fn get_estimate_rate(&self) -> i32 {
unsafe {
gst_video_sys::gst_video_decoder_get_estimate_rate(self.as_ref().to_glib_none().0)
}
}
fn get_max_decode_time(&self, frame: &VideoCodecFrame) -> gst::ClockTimeDiff {
unsafe {
gst_video_sys::gst_video_decoder_get_max_decode_time(self.as_ref().to_glib_none().0, frame.to_glib_none().0)
}
}
fn get_max_errors(&self) -> i32 {
unsafe {
gst_video_sys::gst_video_decoder_get_max_errors(self.as_ref().to_glib_none().0)
}
}
fn get_needs_format(&self) -> bool {
unsafe {
from_glib(gst_video_sys::gst_video_decoder_get_needs_format(self.as_ref().to_glib_none().0))
}
}
fn get_packetized(&self) -> bool {
unsafe {
from_glib(gst_video_sys::gst_video_decoder_get_packetized(self.as_ref().to_glib_none().0))
}
}
fn get_pending_frame_size(&self) -> usize {
unsafe {
gst_video_sys::gst_video_decoder_get_pending_frame_size(self.as_ref().to_glib_none().0)
}
}
fn get_qos_proportion(&self) -> f64 {
unsafe {
gst_video_sys::gst_video_decoder_get_qos_proportion(self.as_ref().to_glib_none().0)
}
}
fn merge_tags(&self, tags: Option<&gst::TagList>, mode: gst::TagMergeMode) {
unsafe {
gst_video_sys::gst_video_decoder_merge_tags(self.as_ref().to_glib_none().0, tags.to_glib_none().0, mode.to_glib());
}
}
fn proxy_getcaps(&self, caps: Option<&gst::Caps>, filter: Option<&gst::Caps>) -> Option<gst::Caps> {
unsafe {
from_glib_full(gst_video_sys::gst_video_decoder_proxy_getcaps(self.as_ref().to_glib_none().0, caps.to_glib_none().0, filter.to_glib_none().0))
}
}
fn set_estimate_rate(&self, enabled: bool) {
unsafe {
gst_video_sys::gst_video_decoder_set_estimate_rate(self.as_ref().to_glib_none().0, enabled.to_glib());
}
}
fn set_max_errors(&self, num: i32) {
unsafe {
gst_video_sys::gst_video_decoder_set_max_errors(self.as_ref().to_glib_none().0, num);
}
}
fn set_needs_format(&self, enabled: bool) {
unsafe {
gst_video_sys::gst_video_decoder_set_needs_format(self.as_ref().to_glib_none().0, enabled.to_glib());
}
}
fn set_packetized(&self, packetized: bool) {
unsafe {
gst_video_sys::gst_video_decoder_set_packetized(self.as_ref().to_glib_none().0, packetized.to_glib());
}
}
fn set_use_default_pad_acceptcaps(&self, use_: bool) {
unsafe {
gst_video_sys::gst_video_decoder_set_use_default_pad_acceptcaps(self.as_ref().to_glib_none().0, use_.to_glib());
}
}
}
......@@ -17,6 +17,7 @@ extern crate gobject_sys;
#[macro_use]
extern crate gstreamer as gst;
extern crate gstreamer_base as gst_base;
extern crate gstreamer_base_sys as gst_base_sys;
extern crate gstreamer_sys as gst_sys;
extern crate gstreamer_video_sys as gst_video_sys;
......@@ -70,6 +71,13 @@ mod video_time_code_interval;
#[cfg(any(feature = "v1_12", feature = "dox"))]
pub use video_time_code_interval::VideoTimeCodeInterval;
mod video_codec_frame;
mod video_decoder;
pub use video_codec_frame::VideoCodecFrame;
pub mod video_codec_state;
pub use video_codec_state::VideoCodecState;
mod utils;
// Re-export all the traits in a prelude module, so that applications
// can always "use gst::prelude::*" without getting conflicts
pub mod prelude {
......@@ -77,5 +85,9 @@ pub mod prelude {
pub use gst::prelude::*;
pub use auto::traits::*;
pub use video_decoder::VideoDecoderExtManual;
pub use video_overlay::VideoOverlayExtManual;
}
#[cfg(feature = "subclassing")]
pub mod subclass;
// Copyright (C) 2019 Philippe Normand <philn@igalia.com>
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#![cfg_attr(feature = "cargo-clippy", allow(cast_ptr_alignment))]
pub mod video_decoder;
pub mod prelude {
pub use super::video_decoder::{VideoDecoderImpl, VideoDecoderImplExt};
}
This diff is collapsed.
// Copyright (C) 2019 Guillaume Desmottes <guillaume.desmottes@collabora.com>
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
pub trait HasStreamLock {
fn get_stream_lock(&self) -> *mut glib_sys::GRecMutex;
fn get_element_as_ptr(&self) -> *const gst_sys::GstElement;
}
// Copyright (C) 2017 Thibault Saunier <tsaunier@gnome.org>
// Copyright (C) 2019 Guillaume Desmottes <guillaume.desmottes@collabora.com>
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use glib::translate::*;
use gst;
use gst::miniobject::MiniObject;
use gst_video_sys;
use std::fmt;
use std::mem;
use utils::HasStreamLock;
use VideoCodecFrameFlags;
pub struct VideoCodecFrame<'a> {
frame: *mut gst_video_sys::GstVideoCodecFrame,
/* GstVideoCodecFrame API isn't safe so protect the frame using the
* element (decoder or encoder) stream lock */
element: &'a dyn HasStreamLock,
}
#[doc(hidden)]
impl<'a> ::glib::translate::ToGlibPtr<'a, *mut gst_video_sys::GstVideoCodecFrame>
for VideoCodecFrame<'a>
{
type Storage = &'a Self;
fn to_glib_none(
&'a self,
) -> ::glib::translate::Stash<'a, *mut gst_video_sys::GstVideoCodecFrame, Self> {
Stash(self.frame, self)
}
fn to_glib_full(&self) -> *mut gst_video_sys::GstVideoCodecFrame {
unimplemented!()
}
}
impl<'a> fmt::Debug for VideoCodecFrame<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
let mut b = f.debug_struct("VideoCodecFrame");
b.field("flags", &self.get_flags())
.field("system_frame_number", &self.get_system_frame_number())
.field("decode_frame_number", &self.get_decode_frame_number())
.field(
"presentation_frame_number",
&self.get_presentation_frame_number(),
)
.field("dts", &self.get_dts())
.field("pts", &self.get_pts())
.field("duration", &self.get_duration())
.field("distance_from_sync", &self.get_distance_from_sync())
.field("input_buffer", &self.get_input_buffer())
.field("output_buffer", &self.get_output_buffer())
.field("deadline", &self.get_deadline());
b.finish()
}
}
impl<'a> VideoCodecFrame<'a> {
// Take ownership of @frame
pub(crate) unsafe fn new<T: HasStreamLock>(
frame: *mut gst_video_sys::GstVideoCodecFrame,
element: &'a T,
) -> Self {
let stream_lock = element.get_stream_lock();
glib_sys::g_rec_mutex_lock(stream_lock);
Self { frame, element }
}
pub fn get_flags(&self) -> VideoCodecFrameFlags {
let flags = unsafe { (*self.to_glib_none().0).flags };
VideoCodecFrameFlags::from_bits_truncate(flags)
}
pub fn set_flags(&self, flags: VideoCodecFrameFlags) {
unsafe { (*self.to_glib_none().0).flags |= flags.bits() }
}
pub fn unset_flags(&self, flags: VideoCodecFrameFlags) {
unsafe { (*self.to_glib_none().0).flags &= !flags.bits() }
}
pub fn get_system_frame_number(&self) -> u32 {
unsafe { (*self.to_glib_none().0).system_frame_number }
}
pub fn get_decode_frame_number(&self) -> u32 {
unsafe { (*self.to_glib_none().0).decode_frame_number }
}
pub fn get_presentation_frame_number(&self) -> u32 {
unsafe { (*self.to_glib_none().0).presentation_frame_number }
}
pub fn get_dts(&self) -> gst::ClockTime {
unsafe { from_glib((*self.to_glib_none().0).dts) }
}
pub fn set_dts(&self, dts: gst::ClockTime) {
unsafe {
(*self.to_glib_none().0).dts = dts.to_glib();
}
}
pub fn get_pts(&self) -> gst::ClockTime {
unsafe { from_glib((*self.to_glib_none().0).pts) }
}
pub fn set_pts(&self, pts: gst::ClockTime) {
unsafe {
(*self.to_glib_none().0).pts = pts.to_glib();
}
}
pub fn get_duration(&self) -> gst::ClockTime {
unsafe { from_glib((*self.to_glib_none().0).duration) }
}
pub fn set_duration(&self, duration: gst::ClockTime) {
unsafe {
(*self.to_glib_none().0).duration = duration.to_glib();
}
}
pub fn get_distance_from_sync(&self) -> i32 {
unsafe { (*self.to_glib_none().0).distance_from_sync }
}
pub fn get_input_buffer(&self) -> Option<&gst::BufferRef> {
unsafe {
let ptr = (*self.to_glib_none().0).input_buffer;
if ptr.is_null() {
None
} else {
Some(gst::BufferRef::from_mut_ptr(ptr))
}
}
}
pub fn get_output_buffer(&self) -> Option<&mut gst::BufferRef> {
unsafe {
let ptr = (*self.to_glib_none().0).output_buffer;
if ptr.is_null() {
None
} else {
let writable: bool = from_glib(gst_sys::gst_mini_object_is_writable(
ptr as *const gst_sys::GstMiniObject,
));
assert!(writable);
Some(gst::BufferRef::from_mut_ptr(ptr))
}
}
}
pub fn set_output_buffer(&self, output_buffer: gst::Buffer) {
unsafe {
let prev = (*self.to_glib_none().0).output_buffer;
if !prev.is_null() {
gst_sys::gst_mini_object_unref(prev as *mut gst_sys::GstMiniObject);
}
let ptr = output_buffer.into_ptr();
let writable: bool = from_glib(gst_sys::gst_mini_object_is_writable(
ptr as *const gst_sys::GstMiniObject,
));
assert!(writable);
(*self.to_glib_none().0).output_buffer = ptr;
}
}
pub fn get_deadline(&self) -> gst::ClockTime {
unsafe { from_glib((*self.to_glib_none().0).deadline) }
}
#[doc(hidden)]
pub unsafe fn into_ptr(self) -> *mut gst_video_sys::GstVideoCodecFrame {
let stream_lock = self.element.get_stream_lock();
glib_sys::g_rec_mutex_unlock(stream_lock);
let ptr = self.to_glib_none().0;
mem::forget(self);
ptr
}
}
impl<'a> Drop for VideoCodecFrame<'a> {
fn drop(&mut self) {
unsafe {
let stream_lock = self.element.get_stream_lock();
glib_sys::g_rec_mutex_unlock(stream_lock);
gst_video_sys::gst_video_codec_frame_unref(self.frame);
}
}
}
// Copyright (C) 2017 Thibault Saunier <tsaunier@gnome.org>
// Copyright (C) 2019 Guillaume Desmottes <guillaume.desmottes@collabora.com>
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use glib::translate::*;
use gst_sys;
use gst_video_sys;
use std::fmt;
use std::marker::PhantomData;
use std::ptr;
use utils::HasStreamLock;
use gst;
use gst::miniobject::MiniObject;
use video_info::VideoInfo;
pub trait VideoCodecStateContext<'a> {
fn get_element(&self) -> Option<&'a dyn HasStreamLock>;
fn get_element_as_ptr(&self) -> *const gst_sys::GstElement;
}
pub struct InNegotiation<'a> {
/* GstVideoCodecState API isn't safe so protect the state using the
* element (decoder or encoder) stream lock */
element: &'a dyn HasStreamLock,
}
pub struct Readable {}
impl<'a> VideoCodecStateContext<'a> for InNegotiation<'a> {
fn get_element(&self) -> Option<&'a dyn HasStreamLock> {
Some(self.element)
}
fn get_element_as_ptr(&self) -> *const gst_sys::GstElement {
self.element.get_element_as_ptr()
}
}
impl<'a> VideoCodecStateContext<'a> for Readable {
fn get_element(&self) -> Option<&'a dyn HasStreamLock> {
None
}
fn get_element_as_ptr(&self) -> *const gst_sys::GstElement {
ptr::null()
}
}
pub struct VideoCodecState<'a, T: VideoCodecStateContext<'a>> {
state: *mut gst_video_sys::GstVideoCodecState,
pub(crate) context: T,
/* FIXME: should not be needed because lifetime is actually used */
phantom: PhantomData<&'a T>,
}
impl<'a, T: VideoCodecStateContext<'a>> fmt::Debug for VideoCodecState<'a, T> {
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
f.debug_struct("VideoCodecState")
.field("info", &self.get_info())
.field("caps", &self.get_caps())
.field("codec_data", &self.get_codec_data())
.field("allocation_caps", &self.get_allocation_caps())
.finish()
}
}
impl<'a> VideoCodecState<'a, Readable> {
// Take ownership of @state
pub(crate) unsafe fn new(state: *mut gst_video_sys::GstVideoCodecState) -> Self {
Self {
state,
context: Readable {},
phantom: PhantomData,
}
}
}
impl<'a> VideoCodecState<'a, InNegotiation<'a>> {
// Take ownership of @state
pub(crate) unsafe fn new<T: HasStreamLock>(
state: *mut gst_video_sys::GstVideoCodecState,
element: &'a T,
) -> Self {
let stream_lock = element.get_stream_lock();
glib_sys::g_rec_mutex_lock(stream_lock);
Self {
state,
context: InNegotiation { element },
phantom: PhantomData,
}
}
}
impl<'a, T: VideoCodecStateContext<'a>> VideoCodecState<'a, T> {
pub fn get_info(&self) -> VideoInfo {