diff --git a/subprojects/gst-plugins-bad/sys/applemedia/coremediabuffer.h b/subprojects/gst-plugins-bad/sys/applemedia/coremediabuffer.h index 69299b7324a0a30c2e4cf72fd82283a846a971cf..876f663c8f644be8c98644bb82ea3f326a7906c7 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/coremediabuffer.h +++ b/subprojects/gst-plugins-bad/sys/applemedia/coremediabuffer.h @@ -24,7 +24,7 @@ #include #include "videotexturecache.h" -#include "CoreMedia/CoreMedia.h" +#include G_BEGIN_DECLS diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c b/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c index a17b762a55a07acd8b6ab031fa0196d6ddfb2fce..76a268aeed4841b79ef4d6319c8de92a9c7a0561 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c @@ -57,6 +57,7 @@ enum /* leave some headroom for new GstVideoCodecFrameFlags flags */ VTDEC_FRAME_FLAG_SKIP = (1 << 10), VTDEC_FRAME_FLAG_DROP = (1 << 11), + VTDEC_FRAME_FLAG_ERROR = (1 << 12), }; static void gst_vtdec_finalize (GObject * object); @@ -101,7 +102,9 @@ static GstStaticPadTemplate gst_vtdec_sink_template = GST_STATIC_CAPS ("video/x-h264, stream-format=avc, alignment=au," " width=(int)[1, MAX], height=(int)[1, MAX];" "video/mpeg, mpegversion=2, systemstream=false, parsed=true;" - "image/jpeg") + "image/jpeg;" + "video/x-prores, variant = { (string)standard, (string)hq, (string)lt," + " (string)proxy, (string)4444, (string)4444xq };") ); /* define EnableHardwareAcceleratedVideoDecoder in < 10.9 */ @@ -114,20 +117,20 @@ const CFStringRef CFSTR ("RequireHardwareAcceleratedVideoDecoder"); #endif -#if defined(APPLEMEDIA_MOLTENVK) -#define VIDEO_SRC_CAPS \ - GST_VIDEO_CAPS_MAKE("NV12") ";" \ - GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\ - "NV12") ", " \ - "texture-target = (string) rectangle ; " \ - GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,\ - "NV12") -#else -#define VIDEO_SRC_CAPS \ - GST_VIDEO_CAPS_MAKE("NV12") ";" \ +#define VIDEO_SRC_CAPS_FORMATS "{ NV12, AYUV64, RGBA64_LE, ARGB64_BE }" + +#define VIDEO_SRC_CAPS_NATIVE \ + GST_VIDEO_CAPS_MAKE(VIDEO_SRC_CAPS_FORMATS) ";" \ GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\ - "NV12") ", " \ + VIDEO_SRC_CAPS_FORMATS) ", " \ "texture-target = (string) rectangle " + +#if defined(APPLEMEDIA_MOLTENVK) +#define VIDEO_SRC_CAPS VIDEO_SRC_CAPS_NATIVE "; " \ + GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, \ + VIDEO_SRC_CAPS_FORMATS) +#else +#define VIDEO_SRC_CAPS VIDEO_SRC_CAPS_NATIVE #endif G_DEFINE_TYPE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER); @@ -234,24 +237,56 @@ gst_vtdec_stop (GstVideoDecoder * decoder) } static void -setup_texture_cache (GstVtdec * vtdec) +setup_texture_cache (GstVtdec * vtdec, GstVideoFormat format) { GstVideoCodecState *output_state; + GST_INFO_OBJECT (vtdec, "setting up texture cache"); output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec)); - gst_video_texture_cache_set_format (vtdec->texture_cache, - GST_VIDEO_FORMAT_NV12, output_state->caps); + gst_video_texture_cache_set_format (vtdec->texture_cache, format, + output_state->caps); gst_video_codec_state_unref (output_state); } +/* + * Unconditionally output a high bit-depth + alpha format when decoding Apple + * ProRes video if downstream supports it. + * TODO: read src_pix_fmt to get the preferred output format + * https://wiki.multimedia.cx/index.php/Apple_ProRes#Frame_header + */ +static GstVideoFormat +get_preferred_video_format (GstStructure * s, gboolean prores) +{ + const GValue *list = gst_structure_get_value (s, "format"); + guint i, size = gst_value_list_get_size (list); + for (i = 0; i < size; i++) { + const GValue *value = gst_value_list_get_value (list, i); + const char *fmt = g_value_get_string (value); + GstVideoFormat vfmt = gst_video_format_from_string (fmt); + switch (vfmt) { + case GST_VIDEO_FORMAT_NV12: + if (!prores) + return vfmt; + break; + case GST_VIDEO_FORMAT_AYUV64: + case GST_VIDEO_FORMAT_ARGB64_BE: + case GST_VIDEO_FORMAT_RGBA64_LE: + if (prores) + return vfmt; + break; + default: + break; + } + } + return GST_VIDEO_FORMAT_UNKNOWN; +} + static gboolean gst_vtdec_negotiate (GstVideoDecoder * decoder) { GstVideoCodecState *output_state = NULL; GstCaps *peercaps = NULL, *caps = NULL, *templcaps = NULL, *prevcaps = NULL; - GstVideoFormat format; - GstStructure *structure; - const gchar *s; + GstVideoFormat format = GST_VIDEO_FORMAT_UNKNOWN; GstVtdec *vtdec; OSStatus err = noErr; GstCapsFeatures *features = NULL; @@ -290,9 +325,30 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder) gst_caps_unref (peercaps); caps = gst_caps_truncate (gst_caps_make_writable (caps)); - structure = gst_caps_get_structure (caps, 0); - s = gst_structure_get_string (structure, "format"); - format = gst_video_format_from_string (s); + + /* Try to use whatever video format downstream prefers */ + { + GstStructure *s = gst_caps_get_structure (caps, 0); + + if (gst_structure_has_field_typed (s, "format", GST_TYPE_LIST)) { + GstStructure *is = gst_caps_get_structure (vtdec->input_state->caps, 0); + const char *name = gst_structure_get_name (is); + format = get_preferred_video_format (s, + g_strcmp0 (name, "video/x-prores") == 0); + } + + if (format == GST_VIDEO_FORMAT_UNKNOWN) { + const char *fmt; + gst_structure_fixate_field (s, "format"); + fmt = gst_structure_get_string (s, "format"); + if (fmt) + format = gst_video_format_from_string (fmt); + else + /* If all fails, just use NV12 */ + format = GST_VIDEO_FORMAT_NV12; + } + } + features = gst_caps_get_features (caps, 0); if (features) features = gst_caps_features_copy (features); @@ -383,7 +439,7 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder) if (!vtdec->texture_cache) { vtdec->texture_cache = gst_video_texture_cache_gl_new (vtdec->ctxh->context); - setup_texture_cache (vtdec); + setup_texture_cache (vtdec, format); } } #if defined(APPLEMEDIA_MOLTENVK) @@ -420,7 +476,7 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder) if (!vtdec->texture_cache) { vtdec->texture_cache = gst_video_texture_cache_vulkan_new (vtdec->device); - setup_texture_cache (vtdec); + setup_texture_cache (vtdec, format); } } #endif @@ -454,6 +510,16 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) cm_format = kCMVideoCodecType_MPEG2Video; } else if (!strcmp (caps_name, "image/jpeg")) { cm_format = kCMVideoCodecType_JPEG; + } else if (!strcmp (caps_name, "video/x-prores")) { + const char *variant = gst_structure_get_string (structure, "variant"); + + if (variant) + cm_format = gst_vtutil_codec_type_from_prores_variant (variant); + + if (cm_format == GST_kCMVideoCodecType_Some_AppleProRes) { + GST_ERROR_OBJECT (vtdec, "Invalid ProRes variant %s", variant); + return FALSE; + } } if (cm_format == kCMVideoCodecType_H264 && state->codec_data == NULL) { @@ -584,11 +650,18 @@ gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format, case GST_VIDEO_FORMAT_NV12: cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; break; - case GST_VIDEO_FORMAT_UYVY: - cv_format = kCVPixelFormatType_422YpCbCr8; + case GST_VIDEO_FORMAT_AYUV64: +/* This is fine for now because Apple only ships LE devices */ +#if G_BYTE_ORDER != G_LITTLE_ENDIAN +#error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE" +#endif + cv_format = kCVPixelFormatType_4444AYpCbCr16; break; - case GST_VIDEO_FORMAT_RGBA: - cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; + case GST_VIDEO_FORMAT_ARGB64_BE: + cv_format = kCVPixelFormatType_64ARGB; + break; + case GST_VIDEO_FORMAT_RGBA64_LE: + cv_format = kCVPixelFormatType_64RGBALE; break; default: g_warn_if_reached (); @@ -929,12 +1002,16 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain, * example) or we're draining/flushing */ if (frame) { - if (flush || frame->flags & VTDEC_FRAME_FLAG_SKIP) + if (frame->flags & VTDEC_FRAME_FLAG_ERROR) { + gst_video_decoder_release_frame (decoder, frame); + ret = GST_FLOW_ERROR; + } else if (flush || frame->flags & VTDEC_FRAME_FLAG_SKIP) { gst_video_decoder_release_frame (decoder, frame); - else if (frame->flags & VTDEC_FRAME_FLAG_DROP) + } else if (frame->flags & VTDEC_FRAME_FLAG_DROP) { gst_video_decoder_drop_frame (decoder, frame); - else + } else { ret = gst_video_decoder_finish_frame (decoder, frame); + } } if (!frame || ret != GST_FLOW_OK) diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c index 9ada8d0a5c028214f1c2afd89fa41ec2332b83fc..ecdc5597f317d50b5507fc917ddc649a3846c75f 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c @@ -35,6 +35,7 @@ #define VTENC_DEFAULT_QUALITY 0.5 #define VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL 0 #define VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL_DURATION 0 +#define VTENC_DEFAULT_PRESERVE_ALPHA TRUE GST_DEBUG_CATEGORY (gst_vtenc_debug); #define GST_CAT_DEFAULT (gst_vtenc_debug) @@ -66,6 +67,12 @@ VTCompressionSessionPrepareToEncodeFrames (VTCompressionSessionRef session) __attribute__ ((weak_import)); #endif +/* This property key is currently completely undocumented. The only way you can + * know about its existence is if Apple tells you. It allows you to tell the + * encoder to not preserve alpha even when outputting alpha formats. */ +const CFStringRef gstVTCodecPropertyKey_PreserveAlphaChannel = +CFSTR ("kVTCodecPropertyKey_PreserveAlphaChannel"); + enum { PROP_0, @@ -75,7 +82,8 @@ enum PROP_REALTIME, PROP_QUALITY, PROP_MAX_KEYFRAME_INTERVAL, - PROP_MAX_KEYFRAME_INTERVAL_DURATION + PROP_MAX_KEYFRAME_INTERVAL_DURATION, + PROP_PRESERVE_ALPHA, }; typedef struct _GstVTEncFrame GstVTEncFrame; @@ -151,7 +159,8 @@ static GstStaticCaps sink_caps = GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ NV12, I420 }")); #else static GstStaticCaps sink_caps = -GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ UYVY, NV12, I420 }")); +GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE + ("{ AYUV64, UYVY, NV12, I420, RGBA64_LE, ARGB64_BE }")); #endif static void @@ -187,10 +196,58 @@ gst_vtenc_base_init (GstVTEncClass * klass) "height", GST_TYPE_INT_RANGE, min_height, max_height, "framerate", GST_TYPE_FRACTION_RANGE, min_fps_n, min_fps_d, max_fps_n, max_fps_d, NULL); - if (codec_details->format_id == kCMVideoCodecType_H264) { - gst_structure_set (gst_caps_get_structure (src_caps, 0), - "stream-format", G_TYPE_STRING, "avc", - "alignment", G_TYPE_STRING, "au", NULL); + + /* Signal our limited interlace support */ + { + G_GNUC_BEGIN_IGNORE_DEPRECATIONS; + GValueArray *arr = g_value_array_new (2); + GValue val = G_VALUE_INIT; + + g_value_init (&val, G_TYPE_STRING); + g_value_set_string (&val, "progressive"); + arr = g_value_array_append (arr, &val); + g_value_set_string (&val, "interleaved"); + arr = g_value_array_append (arr, &val); + G_GNUC_END_IGNORE_DEPRECATIONS; + gst_structure_set_list (gst_caps_get_structure (src_caps, 0), + "interlace-mode", arr); + } + + switch (codec_details->format_id) { + case kCMVideoCodecType_H264: + gst_structure_set (gst_caps_get_structure (src_caps, 0), + "stream-format", G_TYPE_STRING, "avc", + "alignment", G_TYPE_STRING, "au", NULL); + break; + case GST_kCMVideoCodecType_Some_AppleProRes: + if (g_strcmp0 (codec_details->mimetype, "video/x-prores") == 0) { + G_GNUC_BEGIN_IGNORE_DEPRECATIONS; + GValueArray *arr = g_value_array_new (6); + GValue val = G_VALUE_INIT; + + g_value_init (&val, G_TYPE_STRING); + g_value_set_string (&val, "standard"); + arr = g_value_array_append (arr, &val); + g_value_set_string (&val, "4444xq"); + arr = g_value_array_append (arr, &val); + g_value_set_string (&val, "4444"); + arr = g_value_array_append (arr, &val); + g_value_set_string (&val, "hq"); + arr = g_value_array_append (arr, &val); + g_value_set_string (&val, "lt"); + arr = g_value_array_append (arr, &val); + g_value_set_string (&val, "proxy"); + arr = g_value_array_append (arr, &val); + gst_structure_set_list (gst_caps_get_structure (src_caps, 0), + "variant", arr); + g_value_array_free (arr); + g_value_unset (&val); + G_GNUC_END_IGNORE_DEPRECATIONS; + break; + } + /* fall through */ + default: + g_assert_not_reached (); } src_template = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, src_caps); @@ -257,6 +314,12 @@ gst_vtenc_class_init (GstVTEncClass * klass) "Maximum number of nanoseconds between keyframes (0 = no limit)", 0, G_MAXUINT64, VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL_DURATION, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PRESERVE_ALPHA, + g_param_spec_boolean ("preserve-alpha", "Preserve Video Alpha Values", + "Video alpha values (non opaque) need to be perserved.", + VTENC_DEFAULT_PRESERVE_ALPHA, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); } static void @@ -274,6 +337,7 @@ gst_vtenc_init (GstVTEnc * self) self->latency_frames = -1; self->session = NULL; self->profile_level = NULL; + self->have_field_order = TRUE; self->keyframe_props = CFDictionaryCreate (NULL, (const void **) keyframe_props_keys, @@ -463,6 +527,9 @@ gst_vtenc_get_property (GObject * obj, guint prop_id, GValue * value, g_value_set_uint64 (value, gst_vtenc_get_max_keyframe_interval_duration (self)); break; + case PROP_PRESERVE_ALPHA: + g_value_set_boolean (value, self->preserve_alpha); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (obj, prop_id, pspec); break; @@ -495,6 +562,9 @@ gst_vtenc_set_property (GObject * obj, guint prop_id, const GValue * value, gst_vtenc_set_max_keyframe_interval_duration (self, g_value_get_uint64 (value)); break; + case PROP_PRESERVE_ALPHA: + self->preserve_alpha = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (obj, prop_id, pspec); break; @@ -609,7 +679,8 @@ gst_vtenc_profile_level_key (GstVTEnc * self, const gchar * profile, } else if (!strcmp (profile, "main")) { profile = "Main"; } else { - g_assert_not_reached (); + GST_ERROR_OBJECT (self, "invalid profile: %s", profile); + return ret; } if (strlen (level) == 1) { @@ -631,13 +702,45 @@ gst_vtenc_profile_level_key (GstVTEnc * self, const gchar * profile, } static gboolean -gst_vtenc_negotiate_profile_and_level (GstVideoEncoder * enc) +gst_vtenc_negotiate_profile_and_level (GstVTEnc * self, GstStructure * s) +{ + const gchar *profile = gst_structure_get_string (s, "profile"); + const gchar *level = gst_structure_get_string (s, "level"); + + if (self->profile_level) + CFRelease (self->profile_level); + self->profile_level = gst_vtenc_profile_level_key (self, profile, level); + if (self->profile_level == NULL) { + GST_ERROR_OBJECT (self, "unsupported h264 profile '%s' or level '%s'", + profile, level); + return FALSE; + } + + return TRUE; +} + +static gboolean +gst_vtenc_negotiate_prores_variant (GstVTEnc * self, GstStructure * s) +{ + const char *variant = gst_structure_get_string (s, "variant"); + CMVideoCodecType codec_type = + gst_vtutil_codec_type_from_prores_variant (variant); + + if (codec_type == GST_kCMVideoCodecType_Some_AppleProRes) { + GST_ERROR_OBJECT (self, "unsupported prores variant: %s", variant); + return FALSE; + } + + self->specific_format_id = codec_type; + return TRUE; +} + +static gboolean +gst_vtenc_negotiate_specific_format_details (GstVideoEncoder * enc) { GstVTEnc *self = GST_VTENC_CAST (enc); GstCaps *allowed_caps = NULL; gboolean ret = TRUE; - const gchar *profile = NULL; - const gchar *level = NULL; allowed_caps = gst_pad_get_allowed_caps (GST_VIDEO_ENCODER_SRC_PAD (enc)); if (allowed_caps) { @@ -651,17 +754,24 @@ gst_vtenc_negotiate_profile_and_level (GstVideoEncoder * enc) allowed_caps = gst_caps_make_writable (allowed_caps); allowed_caps = gst_caps_fixate (allowed_caps); s = gst_caps_get_structure (allowed_caps, 0); - - profile = gst_structure_get_string (s, "profile"); - level = gst_structure_get_string (s, "level"); - } - - if (self->profile_level) - CFRelease (self->profile_level); - self->profile_level = gst_vtenc_profile_level_key (self, profile, level); - if (self->profile_level == NULL) { - GST_ERROR_OBJECT (enc, "invalid profile and level"); - goto fail; + switch (self->details->format_id) { + case kCMVideoCodecType_H264: + self->specific_format_id = kCMVideoCodecType_H264; + if (!gst_vtenc_negotiate_profile_and_level (self, s)) + goto fail; + break; + case GST_kCMVideoCodecType_Some_AppleProRes: + if (g_strcmp0 (self->details->mimetype, "video/x-prores") != 0) { + GST_ERROR_OBJECT (self, "format_id == %i mimetype must be Apple " + "ProRes", GST_kCMVideoCodecType_Some_AppleProRes); + goto fail; + } + if (!gst_vtenc_negotiate_prores_variant (self, s)) + goto fail; + break; + default: + g_assert_not_reached (); + } } out: @@ -695,7 +805,7 @@ gst_vtenc_set_format (GstVideoEncoder * enc, GstVideoCodecState * state) gst_vtenc_destroy_session (self, &self->session); GST_OBJECT_UNLOCK (self); - gst_vtenc_negotiate_profile_and_level (enc); + gst_vtenc_negotiate_specific_format_details (enc); session = gst_vtenc_create_session (self); GST_OBJECT_LOCK (self); @@ -735,36 +845,48 @@ gst_vtenc_negotiate_downstream (GstVTEnc * self, CMSampleBufferRef sbuf) "framerate", GST_TYPE_FRACTION, self->negotiated_fps_n, self->negotiated_fps_d, NULL); - if (self->details->format_id == kCMVideoCodecType_H264) { - CMFormatDescriptionRef fmt; - CFDictionaryRef atoms; - CFStringRef avccKey; - CFDataRef avcc; - guint8 *codec_data; - gsize codec_data_size; - GstBuffer *codec_data_buf; - guint8 sps[3]; - - fmt = CMSampleBufferGetFormatDescription (sbuf); - atoms = CMFormatDescriptionGetExtension (fmt, - kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms); - avccKey = CFStringCreateWithCString (NULL, "avcC", kCFStringEncodingUTF8); - avcc = CFDictionaryGetValue (atoms, avccKey); - CFRelease (avccKey); - codec_data_size = CFDataGetLength (avcc); - codec_data = g_malloc (codec_data_size); - CFDataGetBytes (avcc, CFRangeMake (0, codec_data_size), codec_data); - codec_data_buf = gst_buffer_new_wrapped (codec_data, codec_data_size); - - gst_structure_set (s, "codec_data", GST_TYPE_BUFFER, codec_data_buf, NULL); - - sps[0] = codec_data[1]; - sps[1] = codec_data[2] & ~0xDF; - sps[2] = codec_data[3]; - - gst_codec_utils_h264_caps_set_level_and_profile (caps, sps, 3); - - gst_buffer_unref (codec_data_buf); + switch (self->details->format_id) { + case kCMVideoCodecType_H264: + { + CMFormatDescriptionRef fmt; + CFDictionaryRef atoms; + CFStringRef avccKey; + CFDataRef avcc; + guint8 *codec_data; + gsize codec_data_size; + GstBuffer *codec_data_buf; + guint8 sps[3]; + + fmt = CMSampleBufferGetFormatDescription (sbuf); + atoms = CMFormatDescriptionGetExtension (fmt, + kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms); + avccKey = CFStringCreateWithCString (NULL, "avcC", kCFStringEncodingUTF8); + avcc = CFDictionaryGetValue (atoms, avccKey); + CFRelease (avccKey); + codec_data_size = CFDataGetLength (avcc); + codec_data = g_malloc (codec_data_size); + CFDataGetBytes (avcc, CFRangeMake (0, codec_data_size), codec_data); + codec_data_buf = gst_buffer_new_wrapped (codec_data, codec_data_size); + + gst_structure_set (s, "codec_data", GST_TYPE_BUFFER, codec_data_buf, + NULL); + + sps[0] = codec_data[1]; + sps[1] = codec_data[2] & ~0xDF; + sps[2] = codec_data[3]; + + gst_codec_utils_h264_caps_set_level_and_profile (caps, sps, 3); + + gst_buffer_unref (codec_data_buf); + } + break; + case GST_kCMVideoCodecType_Some_AppleProRes: + gst_structure_set (s, "variant", G_TYPE_STRING, + gst_vtutil_codec_type_to_prores_variant (self->specific_format_id), + NULL); + break; + default: + g_assert_not_reached (); } state = @@ -821,11 +943,120 @@ gst_vtenc_flush (GstVideoEncoder * enc) return (ret == GST_FLOW_OK); } +static void +gst_vtenc_set_colorimetry (GstVTEnc * self, VTCompressionSessionRef session) +{ + OSStatus status; + CFStringRef primaries = NULL, transfer = NULL, matrix = NULL; + GstVideoColorimetry cm = GST_VIDEO_INFO_COLORIMETRY (&self->video_info); + + /* + * https://developer.apple.com/documentation/corevideo/cvimagebuffer/image_buffer_ycbcr_matrix_constants + */ + switch (cm.matrix) { + case GST_VIDEO_COLOR_MATRIX_BT709: + matrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2; + break; + case GST_VIDEO_COLOR_MATRIX_BT601: + matrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4; + break; + case GST_VIDEO_COLOR_MATRIX_SMPTE240M: + matrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995; + break; + case GST_VIDEO_COLOR_MATRIX_BT2020: + matrix = kCVImageBufferYCbCrMatrix_ITU_R_2020; + break; + default: + GST_WARNING_OBJECT (self, "Unsupported color matrix %u", cm.matrix); + } + + /* + * https://developer.apple.com/documentation/corevideo/cvimagebuffer/image_buffer_transfer_function_constants + */ + switch (cm.transfer) { + case GST_VIDEO_TRANSFER_BT709: + case GST_VIDEO_TRANSFER_BT601: + case GST_VIDEO_TRANSFER_UNKNOWN: + transfer = kCVImageBufferTransferFunction_ITU_R_709_2; + break; + case GST_VIDEO_TRANSFER_SMPTE240M: + transfer = kCVImageBufferTransferFunction_SMPTE_240M_1995; + break; + case GST_VIDEO_TRANSFER_BT2020_12: + transfer = kCVImageBufferTransferFunction_ITU_R_2020; + break; + case GST_VIDEO_TRANSFER_SRGB: + if (__builtin_available (macOS 10.13, *)) + transfer = kCVImageBufferTransferFunction_sRGB; + else + GST_WARNING_OBJECT (self, "macOS version is too old, the sRGB transfer " + "function is not available"); + break; + case GST_VIDEO_TRANSFER_SMPTE2084: + if (__builtin_available (macOS 10.13, *)) + transfer = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ; + else + GST_WARNING_OBJECT (self, "macOS version is too old, the SMPTE2084 " + "transfer function is not available"); + break; + default: + GST_WARNING_OBJECT (self, "Unsupported color transfer %u", cm.transfer); + } + + /* + * https://developer.apple.com/documentation/corevideo/cvimagebuffer/image_buffer_color_primaries_constants + */ + switch (cm.primaries) { + case GST_VIDEO_COLOR_PRIMARIES_BT709: + primaries = kCVImageBufferColorPrimaries_ITU_R_709_2; + break; + case GST_VIDEO_COLOR_PRIMARIES_SMPTE170M: + case GST_VIDEO_COLOR_PRIMARIES_SMPTE240M: + primaries = kCVImageBufferColorPrimaries_SMPTE_C; + break; + case GST_VIDEO_COLOR_PRIMARIES_BT2020: + primaries = kCVImageBufferColorPrimaries_ITU_R_2020; + break; + case GST_VIDEO_COLOR_PRIMARIES_SMPTERP431: + primaries = kCVImageBufferColorPrimaries_DCI_P3; + break; + case GST_VIDEO_COLOR_PRIMARIES_SMPTEEG432: + primaries = kCVImageBufferColorPrimaries_P3_D65; + break; + case GST_VIDEO_COLOR_PRIMARIES_EBU3213: + primaries = kCVImageBufferColorPrimaries_EBU_3213; + break; + default: + GST_WARNING_OBJECT (self, "Unsupported color primaries %u", cm.primaries); + } + + if (primaries) { + status = VTSessionSetProperty (session, + kVTCompressionPropertyKey_ColorPrimaries, primaries); + GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_ColorPrimaries =>" + "%d", status); + } + + if (transfer) { + status = VTSessionSetProperty (session, + kVTCompressionPropertyKey_TransferFunction, transfer); + GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_TransferFunction =>" + "%d", status); + } + + if (matrix) { + status = VTSessionSetProperty (session, + kVTCompressionPropertyKey_YCbCrMatrix, matrix); + GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_YCbCrMatrix => %d", + status); + } +} + static VTCompressionSessionRef gst_vtenc_create_session (GstVTEnc * self) { VTCompressionSessionRef session = NULL; - CFMutableDictionaryRef encoder_spec = NULL, pb_attrs; + CFMutableDictionaryRef encoder_spec = NULL, pb_attrs = NULL; OSStatus status; #if !HAVE_IOS @@ -843,16 +1074,21 @@ gst_vtenc_create_session (GstVTEnc * self) TRUE); #endif - pb_attrs = CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks, - &kCFTypeDictionaryValueCallBacks); - gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferWidthKey, - self->negotiated_width); - gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferHeightKey, - self->negotiated_height); + if (self->profile_level) { + pb_attrs = CFDictionaryCreateMutable (NULL, 0, + &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferWidthKey, + self->negotiated_width); + gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferHeightKey, + self->negotiated_height); + } + + /* This was set in gst_vtenc_negotiate_specific_format_details() */ + g_assert_cmpint (self->specific_format_id, !=, 0); status = VTCompressionSessionCreate (NULL, self->negotiated_width, self->negotiated_height, - self->details->format_id, encoder_spec, pb_attrs, NULL, + self->specific_format_id, encoder_spec, pb_attrs, NULL, gst_vtenc_enqueue_buffer, self, &session); GST_INFO_OBJECT (self, "VTCompressionSessionCreate for %d x %d => %d", self->negotiated_width, self->negotiated_height, (int) status); @@ -862,26 +1098,81 @@ gst_vtenc_create_session (GstVTEnc * self) goto beach; } - gst_vtenc_session_configure_expected_framerate (self, session, - (gdouble) self->negotiated_fps_n / (gdouble) self->negotiated_fps_d); + if (self->profile_level) { + gst_vtenc_session_configure_expected_framerate (self, session, + (gdouble) self->negotiated_fps_n / (gdouble) self->negotiated_fps_d); + + /* + * https://developer.apple.com/documentation/videotoolbox/vtcompressionsession/compression_properties/profile_and_level_constants + */ + status = VTSessionSetProperty (session, + kVTCompressionPropertyKey_ProfileLevel, self->profile_level); + GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_ProfileLevel => %d", + (int) status); - status = VTSessionSetProperty (session, - kVTCompressionPropertyKey_ProfileLevel, self->profile_level); - GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_ProfileLevel => %d", - (int) status); + status = VTSessionSetProperty (session, + kVTCompressionPropertyKey_AllowTemporalCompression, kCFBooleanTrue); + GST_DEBUG_OBJECT (self, + "kVTCompressionPropertyKey_AllowTemporalCompression => %d", + (int) status); - status = VTSessionSetProperty (session, - kVTCompressionPropertyKey_AllowTemporalCompression, kCFBooleanTrue); - GST_DEBUG_OBJECT (self, - "kVTCompressionPropertyKey_AllowTemporalCompression => %d", (int) status); + gst_vtenc_session_configure_max_keyframe_interval (self, session, + self->max_keyframe_interval); + gst_vtenc_session_configure_max_keyframe_interval_duration (self, session, + self->max_keyframe_interval_duration / ((gdouble) GST_SECOND)); - gst_vtenc_session_configure_max_keyframe_interval (self, session, - self->max_keyframe_interval); - gst_vtenc_session_configure_max_keyframe_interval_duration (self, session, - self->max_keyframe_interval_duration / ((gdouble) GST_SECOND)); + gst_vtenc_session_configure_bitrate (self, session, + gst_vtenc_get_bitrate (self)); + } + + /* Force encoder to not preserve alpha with 4444(XQ) ProRes formats if + * requested */ + if (!self->preserve_alpha && + (self->specific_format_id == kCMVideoCodecType_AppleProRes4444XQ || + self->specific_format_id == kCMVideoCodecType_AppleProRes4444)) { + status = VTSessionSetProperty (session, + gstVTCodecPropertyKey_PreserveAlphaChannel, CFSTR ("NO")); + GST_DEBUG_OBJECT (self, "kVTCodecPropertyKey_PreserveAlphaChannel => %d", + (int) status); + } + + gst_vtenc_set_colorimetry (self, session); + + /* Interlacing */ + switch (GST_VIDEO_INFO_INTERLACE_MODE (&self->video_info)) { + case GST_VIDEO_INTERLACE_MODE_PROGRESSIVE: + gst_vtenc_session_configure_property_int (self, session, + kVTCompressionPropertyKey_FieldCount, 1); + break; + case GST_VIDEO_INTERLACE_MODE_INTERLEAVED: + gst_vtenc_session_configure_property_int (self, session, + kVTCompressionPropertyKey_FieldCount, 2); + switch (GST_VIDEO_INFO_FIELD_ORDER (&self->video_info)) { + case GST_VIDEO_FIELD_ORDER_TOP_FIELD_FIRST: + status = VTSessionSetProperty (session, + kVTCompressionPropertyKey_FieldDetail, + kCMFormatDescriptionFieldDetail_TemporalTopFirst); + GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_FieldDetail " + "TemporalTopFirst => %d", (int) status); + break; + case GST_VIDEO_FIELD_ORDER_BOTTOM_FIELD_FIRST: + status = VTSessionSetProperty (session, + kVTCompressionPropertyKey_FieldDetail, + kCMFormatDescriptionFieldDetail_TemporalBottomFirst); + GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_FieldDetail " + "TemporalBottomFirst => %d", (int) status); + break; + case GST_VIDEO_FIELD_ORDER_UNKNOWN: + GST_INFO_OBJECT (self, "Unknown field order for interleaved content, " + "will check first buffer"); + self->have_field_order = FALSE; + } + break; + default: + /* Caps negotiation should prevent this */ + g_assert_not_reached (); + } - gst_vtenc_session_configure_bitrate (self, session, - gst_vtenc_get_bitrate (self)); gst_vtenc_session_configure_realtime (self, session, gst_vtenc_get_realtime (self)); gst_vtenc_session_configure_allow_frame_reordering (self, session, @@ -906,7 +1197,8 @@ gst_vtenc_create_session (GstVTEnc * self) beach: if (encoder_spec) CFRelease (encoder_spec); - CFRelease (pb_attrs); + if (pb_attrs) + CFRelease (pb_attrs); return session; } @@ -1133,6 +1425,33 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) else duration = kCMTimeInvalid; + /* If we don't have field order, we need to pick it up from the first buffer + * that has that information. The encoder session also cannot be reconfigured + * with a new field detail after it has been set, so we encode mixed streams + * with whatever the first buffer's field order is. */ + if (!self->have_field_order) { + CFStringRef field_detail = NULL; + + if (GST_VIDEO_BUFFER_IS_TOP_FIELD (frame->input_buffer)) + field_detail = kCMFormatDescriptionFieldDetail_TemporalTopFirst; + else if (GST_VIDEO_BUFFER_IS_BOTTOM_FIELD (frame->input_buffer)) + field_detail = kCMFormatDescriptionFieldDetail_TemporalBottomFirst; + + if (field_detail) { + vt_status = VTSessionSetProperty (self->session, + kVTCompressionPropertyKey_FieldDetail, field_detail); + GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_FieldDetail => %d", + (int) vt_status); + } else { + GST_WARNING_OBJECT (self, "have interlaced content, but don't know field " + "order yet, skipping buffer"); + gst_video_codec_frame_unref (frame); + return GST_FLOW_OK; + } + + self->have_field_order = TRUE; + } + meta = gst_buffer_get_core_media_meta (frame->input_buffer); if (meta != NULL) { pbuf = gst_core_media_buffer_get_pixel_buffer (frame->input_buffer); @@ -1157,18 +1476,21 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) pixel_format_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; break; default: - goto cv_error; + g_assert_not_reached (); } if (!gst_video_frame_map (&inframe, &self->video_info, frame->input_buffer, - GST_MAP_READ)) + GST_MAP_READ)) { + GST_ERROR_OBJECT (self, "failed to map input buffer"); goto cv_error; + } cv_ret = CVPixelBufferCreate (NULL, self->negotiated_width, self->negotiated_height, pixel_format_type, NULL, &pbuf); if (cv_ret != kCVReturnSuccess) { + GST_ERROR_OBJECT (self, "CVPixelBufferCreate failed: %i", cv_ret); gst_video_frame_unmap (&inframe); goto cv_error; } @@ -1177,6 +1499,7 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) gst_core_video_buffer_new ((CVBufferRef) pbuf, &self->video_info, NULL); if (!gst_video_frame_map (&outframe, &self->video_info, outbuf, GST_MAP_WRITE)) { + GST_ERROR_OBJECT (self, "Failed to map output buffer"); gst_video_frame_unmap (&inframe); gst_buffer_unref (outbuf); CVPixelBufferRelease (pbuf); @@ -1184,6 +1507,7 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) } if (!gst_video_frame_copy (&outframe, &inframe)) { + GST_ERROR_OBJECT (self, "Failed to copy output frame"); gst_video_frame_unmap (&inframe); gst_buffer_unref (outbuf); CVPixelBufferRelease (pbuf); @@ -1200,8 +1524,10 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) CVReturn cv_ret; vframe = gst_vtenc_frame_new (frame->input_buffer, &self->video_info); - if (!vframe) + if (!vframe) { + GST_ERROR_OBJECT (self, "Failed to create a new input frame"); goto cv_error; + } { const size_t num_planes = GST_VIDEO_FRAME_N_PLANES (&vframe->videoframe); @@ -1224,6 +1550,19 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) } switch (GST_VIDEO_INFO_FORMAT (&self->video_info)) { + case GST_VIDEO_FORMAT_ARGB64_BE: + pixel_format_type = kCVPixelFormatType_64ARGB; + break; + case GST_VIDEO_FORMAT_AYUV64: +/* This is fine for now because Apple only ships LE devices */ +#if G_BYTE_ORDER != G_LITTLE_ENDIAN +#error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE" +#endif + pixel_format_type = kCVPixelFormatType_4444AYpCbCr16; + break; + case GST_VIDEO_FORMAT_RGBA64_LE: + pixel_format_type = kCVPixelFormatType_64RGBALE; + break; case GST_VIDEO_FORMAT_I420: pixel_format_type = kCVPixelFormatType_420YpCbCr8Planar; break; @@ -1234,8 +1573,7 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) pixel_format_type = kCVPixelFormatType_422YpCbCr8; break; default: - gst_vtenc_frame_free (vframe); - goto cv_error; + g_assert_not_reached (); } cv_ret = CVPixelBufferCreateWithPlanarBytes (NULL, @@ -1250,6 +1588,8 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) plane_bytes_per_row, gst_pixel_buffer_release_cb, vframe, NULL, &pbuf); if (cv_ret != kCVReturnSuccess) { + GST_ERROR_OBJECT (self, "CVPixelBufferCreateWithPlanarBytes failed: %i", + cv_ret); gst_vtenc_frame_free (vframe); goto cv_error; } @@ -1458,6 +1798,8 @@ static const GstVTEncoderDetails gst_vtenc_codecs[] = { #ifndef HAVE_IOS {"H.264 (HW only)", "h264_hw", "video/x-h264", kCMVideoCodecType_H264, TRUE}, #endif + {"Apple ProRes", "prores", "video/x-prores", + GST_kCMVideoCodecType_Some_AppleProRes, FALSE}, }; void diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.h b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.h index 4c6100016f321b487c3c21209c50e385219609b1..073ba75792282d30075987b3fe946fa2ab7da1dd 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.h +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.h @@ -58,6 +58,7 @@ struct _GstVTEnc const GstVTEncoderDetails * details; + CMVideoCodecType specific_format_id; CFStringRef profile_level; guint bitrate; gboolean allow_frame_reordering; @@ -66,6 +67,7 @@ struct _GstVTEnc gint max_keyframe_interval; GstClockTime max_keyframe_interval_duration; gint latency_frames; + gboolean preserve_alpha; gboolean dump_properties; gboolean dump_attributes; @@ -74,6 +76,7 @@ struct _GstVTEnc gint negotiated_fps_n, negotiated_fps_d; gint caps_width, caps_height; gint caps_fps_n, caps_fps_d; + gboolean have_field_order; GstVideoCodecState *input_state; GstVideoInfo video_info; VTCompressionSessionRef session; diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtutil.c b/subprojects/gst-plugins-bad/sys/applemedia/vtutil.c index 6c4441580eab2c310505c5d335069252478dbb24..086c2e9d6d307ba71c52f161d80a98ecdc6dddd7 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtutil.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtutil.c @@ -96,3 +96,42 @@ gst_vtutil_dict_set_object (CFMutableDictionaryRef dict, CFStringRef key, CFDictionarySetValue (dict, key, value); CFRelease (value); } + +CMVideoCodecType +gst_vtutil_codec_type_from_prores_variant (const char *variant) +{ + if (g_strcmp0 (variant, "standard") == 0) + return kCMVideoCodecType_AppleProRes422; + else if (g_strcmp0 (variant, "4444xq") == 0) + return kCMVideoCodecType_AppleProRes4444XQ; + else if (g_strcmp0 (variant, "4444") == 0) + return kCMVideoCodecType_AppleProRes4444; + else if (g_strcmp0 (variant, "hq") == 0) + return kCMVideoCodecType_AppleProRes422HQ; + else if (g_strcmp0 (variant, "lt") == 0) + return kCMVideoCodecType_AppleProRes422LT; + else if (g_strcmp0 (variant, "proxy") == 0) + return kCMVideoCodecType_AppleProRes422Proxy; + return GST_kCMVideoCodecType_Some_AppleProRes; +} + +const char * +gst_vtutil_codec_type_to_prores_variant (CMVideoCodecType codec_type) +{ + switch (codec_type) { + case kCMVideoCodecType_AppleProRes422: + return "standard"; + case kCMVideoCodecType_AppleProRes4444XQ: + return "4444xq"; + case kCMVideoCodecType_AppleProRes4444: + return "4444"; + case kCMVideoCodecType_AppleProRes422HQ: + return "hq"; + case kCMVideoCodecType_AppleProRes422LT: + return "lt"; + case kCMVideoCodecType_AppleProRes422Proxy: + return "proxy"; + default: + g_assert_not_reached (); + } +} diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtutil.h b/subprojects/gst-plugins-bad/sys/applemedia/vtutil.h index 4aa974beca1941c5eb5a22d873a6c3d5af966d56..9e65d258a62cff3fe1f2584e9a4bba3379b8e6e7 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtutil.h +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtutil.h @@ -22,6 +22,12 @@ #include #include +#include + +/* Some formats such as Apple ProRes have separate codec type mappings for all + * variants / profiles, and we don't want to instantiate separate elements for + * each variant, so we use a dummy type for details->format_id */ +#define GST_kCMVideoCodecType_Some_AppleProRes 1 G_BEGIN_DECLS @@ -38,6 +44,9 @@ void gst_vtutil_dict_set_data (CFMutableDictionaryRef dict, void gst_vtutil_dict_set_object (CFMutableDictionaryRef dict, CFStringRef key, CFTypeRef * value); +CMVideoCodecType gst_vtutil_codec_type_from_prores_variant (const char * variant); +const char * gst_vtutil_codec_type_to_prores_variant (CMVideoCodecType codec_type); + G_END_DECLS #endif /* __GST_VTUTIL_H__ */