From fe56af607b7e8ceb265f30b1806833048fbbce98 Mon Sep 17 00:00:00 2001 From: Ming Qian Date: Tue, 26 Oct 2021 10:03:42 +0800 Subject: [PATCH 1/2] v4l2videodec : refactor the setup process of capture v4l2videodec do some refactoring so that it can support dynamic resolution change event. 1.wrap the setup process of capture as a function, as decoder need setup the capture again when dynamic resolution change event is received. 2.move the function "remove_padding" Part-of: --- .../gst-plugins-good/sys/v4l2/gstv4l2object.c | 5 +- .../gst-plugins-good/sys/v4l2/gstv4l2object.h | 1 + .../sys/v4l2/gstv4l2videodec.c | 288 ++++++++++-------- 3 files changed, 161 insertions(+), 133 deletions(-) diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2object.c b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2object.c index 399abab88ad..18665168cab 100644 --- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2object.c +++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2object.c @@ -546,9 +546,6 @@ gst_v4l2_object_new (GstElement * element, return v4l2object; } -static gboolean gst_v4l2_object_clear_format_list (GstV4l2Object * v4l2object); - - void gst_v4l2_object_destroy (GstV4l2Object * v4l2object) { @@ -574,7 +571,7 @@ gst_v4l2_object_destroy (GstV4l2Object * v4l2object) } -static gboolean +gboolean gst_v4l2_object_clear_format_list (GstV4l2Object * v4l2object) { g_slist_foreach (v4l2object->formats, (GFunc) g_free, NULL); diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2object.h b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2object.h index ed32d8ee9a3..98ed586adc7 100644 --- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2object.h +++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2object.h @@ -277,6 +277,7 @@ gboolean gst_v4l2_object_get_property_helper (GstV4l2Object *v4l2objec gboolean gst_v4l2_object_open (GstV4l2Object * v4l2object, GstV4l2Error * error); gboolean gst_v4l2_object_open_shared (GstV4l2Object * v4l2object, GstV4l2Object * other); gboolean gst_v4l2_object_close (GstV4l2Object * v4l2object); +gboolean gst_v4l2_object_clear_format_list (GstV4l2Object * v4l2object); /* probing */ diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c index abd01333cbe..c2aa0d114d6 100644 --- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c +++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c @@ -498,6 +498,146 @@ check_system_frame_number_too_old (guint32 current, guint32 old) return FALSE; } +static gboolean +gst_v4l2_video_remove_padding (GstCapsFeatures * features, + GstStructure * structure, gpointer user_data) +{ + GstV4l2VideoDec *self = GST_V4L2_VIDEO_DEC (user_data); + GstVideoAlignment *align = &self->v4l2capture->align; + GstVideoInfo *info = &self->v4l2capture->info; + int width, height; + + if (!gst_structure_get_int (structure, "width", &width)) + return TRUE; + + if (!gst_structure_get_int (structure, "height", &height)) + return TRUE; + + if (align->padding_left != 0 || align->padding_top != 0 || + height != info->height + align->padding_bottom) + return TRUE; + + if (height == info->height + align->padding_bottom) { + /* Some drivers may round up width to the padded with */ + if (width == info->width + align->padding_right) + gst_structure_set (structure, + "width", G_TYPE_INT, width - align->padding_right, + "height", G_TYPE_INT, height - align->padding_bottom, NULL); + /* Some drivers may keep visible width and only round up bytesperline */ + else if (width == info->width) + gst_structure_set (structure, + "height", G_TYPE_INT, height - align->padding_bottom, NULL); + } + + return TRUE; +} + +static GstFlowReturn +gst_v4l2_video_dec_setup_capture (GstVideoDecoder * decoder) +{ + GstV4l2VideoDec *self = GST_V4L2_VIDEO_DEC (decoder); + GstV4l2Error error = GST_V4L2_ERROR_INIT; + GstVideoInfo info; + GstVideoCodecState *output_state; + GstCaps *acquired_caps, *available_caps, *caps, *filter; + GstStructure *st; + + if (G_UNLIKELY (!GST_V4L2_IS_ACTIVE (self->v4l2capture))) { + /* init capture fps according to output */ + self->v4l2capture->info.fps_d = self->v4l2output->info.fps_d; + self->v4l2capture->info.fps_n = self->v4l2output->info.fps_n; + + /* For decoders G_FMT returns coded size, G_SELECTION returns visible size + * in the compose rectangle. gst_v4l2_object_acquire_format() checks both + * and returns the visible size as with/height and the coded size as + * padding. */ + if (!gst_v4l2_object_acquire_format (self->v4l2capture, &info)) + goto not_negotiated; + + /* gst_v4l2_object_acquire_format() does not set fps, copy from sink */ + info.fps_n = self->v4l2output->info.fps_n; + info.fps_d = self->v4l2output->info.fps_d; + + gst_v4l2_object_clear_format_list (self->v4l2capture); + gst_caps_replace (&self->probed_srccaps, NULL); + self->probed_srccaps = gst_v4l2_object_probe_caps (self->v4l2capture, + gst_v4l2_object_get_raw_caps ()); + /* Create caps from the acquired format, remove the format field */ + acquired_caps = gst_video_info_to_caps (&info); + GST_DEBUG_OBJECT (self, "Acquired caps: %" GST_PTR_FORMAT, acquired_caps); + st = gst_caps_get_structure (acquired_caps, 0); + gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", + NULL); + + /* Probe currently available pixel formats */ + available_caps = gst_caps_copy (self->probed_srccaps); + GST_DEBUG_OBJECT (self, "Available caps: %" GST_PTR_FORMAT, available_caps); + + /* Replace coded size with visible size, we want to negotiate visible size + * with downstream, not coded size. */ + gst_caps_map_in_place (available_caps, gst_v4l2_video_remove_padding, self); + + filter = gst_caps_intersect_full (available_caps, acquired_caps, + GST_CAPS_INTERSECT_FIRST); + GST_DEBUG_OBJECT (self, "Filtered caps: %" GST_PTR_FORMAT, filter); + gst_caps_unref (acquired_caps); + gst_caps_unref (available_caps); + caps = gst_pad_peer_query_caps (decoder->srcpad, filter); + gst_caps_unref (filter); + + GST_DEBUG_OBJECT (self, "Possible decoded caps: %" GST_PTR_FORMAT, caps); + if (gst_caps_is_empty (caps)) { + gst_caps_unref (caps); + goto not_negotiated; + } + + /* Fixate pixel format */ + caps = gst_caps_fixate (caps); + + GST_DEBUG_OBJECT (self, "Chosen decoded caps: %" GST_PTR_FORMAT, caps); + + /* Try to set negotiated format, on success replace acquired format */ + if (gst_v4l2_object_set_format (self->v4l2capture, caps, &error)) + gst_video_info_from_caps (&info, caps); + else + gst_v4l2_clear_error (&error); + gst_caps_unref (caps); + + output_state = gst_video_decoder_set_output_state (decoder, + info.finfo->format, info.width, info.height, self->input_state); + + /* Copy the rest of the information, there might be more in the future */ + output_state->info.interlace_mode = info.interlace_mode; + gst_video_codec_state_unref (output_state); + + if (!gst_video_decoder_negotiate (decoder)) { + if (GST_PAD_IS_FLUSHING (decoder->srcpad)) + goto flushing; + else + goto not_negotiated; + } + + /* Ensure our internal pool is activated */ + if (!gst_buffer_pool_set_active (GST_BUFFER_POOL (self->v4l2capture->pool), + TRUE)) + goto activate_failed; + } + + return GST_FLOW_OK; + +not_negotiated: + GST_ERROR_OBJECT (self, "not negotiated"); + gst_v4l2_error (self, &error); + return GST_FLOW_NOT_NEGOTIATED; +activate_failed: + GST_ELEMENT_ERROR (self, RESOURCE, SETTINGS, + (_("Failed to allocate required memory.")), + ("Buffer pool activation failed")); + return GST_FLOW_ERROR; +flushing: + return GST_FLOW_FLUSHING; +} + static void gst_v4l2_video_dec_loop (GstVideoDecoder * decoder) { @@ -589,46 +729,13 @@ beach: gst_pad_pause_task (decoder->srcpad); } -static gboolean -gst_v4l2_video_remove_padding (GstCapsFeatures * features, - GstStructure * structure, gpointer user_data) -{ - GstV4l2VideoDec *self = GST_V4L2_VIDEO_DEC (user_data); - GstVideoAlignment *align = &self->v4l2capture->align; - GstVideoInfo *info = &self->v4l2capture->info; - int width, height; - - if (!gst_structure_get_int (structure, "width", &width)) - return TRUE; - - if (!gst_structure_get_int (structure, "height", &height)) - return TRUE; - - if (align->padding_left != 0 || align->padding_top != 0 || - height != info->height + align->padding_bottom) - return TRUE; - - if (height == info->height + align->padding_bottom) { - /* Some drivers may round up width to the padded with */ - if (width == info->width + align->padding_right) - gst_structure_set (structure, - "width", G_TYPE_INT, width - align->padding_right, - "height", G_TYPE_INT, height - align->padding_bottom, NULL); - /* Some drivers may keep visible width and only round up bytesperline */ - else if (width == info->width) - gst_structure_set (structure, - "height", G_TYPE_INT, height - align->padding_bottom, NULL); - } - - return TRUE; -} - static GstFlowReturn gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame) { GstV4l2Error error = GST_V4L2_ERROR_INIT; GstV4l2VideoDec *self = GST_V4L2_VIDEO_DEC (decoder); + GstBufferPool *pool = GST_BUFFER_POOL (self->v4l2output->pool); GstFlowReturn ret = GST_FLOW_OK; gboolean processed = FALSE; GstBuffer *tmp; @@ -647,17 +754,21 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder, goto not_negotiated; } - if (G_UNLIKELY (!GST_V4L2_IS_ACTIVE (self->v4l2capture))) { - GstBufferPool *pool = GST_BUFFER_POOL (self->v4l2output->pool); - GstVideoInfo info; - GstVideoCodecState *output_state; + ret = gst_v4l2_video_dec_setup_capture (decoder); + if (ret != GST_FLOW_OK) { + GST_ERROR_OBJECT (decoder, "setup capture fail\n"); + goto not_negotiated; + } + + if (G_UNLIKELY (!gst_buffer_pool_is_active (pool))) { GstBuffer *codec_data; - GstCaps *acquired_caps, *available_caps, *caps, *filter; - GstStructure *st; + GstStructure *config = gst_buffer_pool_get_config (pool); + guint min = MAX (self->v4l2output->min_buffers, + GST_V4L2_MIN_BUFFERS (self->v4l2output)); + guint max = VIDEO_MAX_FRAME; guint32 dummy_frame_number = 0; GST_DEBUG_OBJECT (self, "Sending header"); - codec_data = self->input_state->codec_data; /* We are running in byte-stream mode, so we don't know the headers, but @@ -672,22 +783,16 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder, } /* Ensure input internal pool is active */ - if (!gst_buffer_pool_is_active (pool)) { - GstStructure *config = gst_buffer_pool_get_config (pool); - guint min = MAX (self->v4l2output->min_buffers, - GST_V4L2_MIN_BUFFERS (self->v4l2output)); - guint max = VIDEO_MAX_FRAME; - gst_buffer_pool_config_set_params (config, self->input_state->caps, - self->v4l2output->info.size, min, max); + gst_buffer_pool_config_set_params (config, self->input_state->caps, + self->v4l2output->info.size, min, max); - /* There is no reason to refuse this config */ - if (!gst_buffer_pool_set_config (pool, config)) - goto activate_failed; + /* There is no reason to refuse this config */ + if (!gst_buffer_pool_set_config (pool, config)) + goto activate_failed; - if (!gst_buffer_pool_set_active (pool, TRUE)) - goto activate_failed; - } + if (!gst_buffer_pool_set_active (pool, TRUE)) + goto activate_failed; GST_VIDEO_DECODER_STREAM_UNLOCK (decoder); GST_LOG_OBJECT (decoder, "Passing buffer with system frame number %u", @@ -699,81 +804,6 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder, GST_VIDEO_DECODER_STREAM_LOCK (decoder); gst_buffer_unref (codec_data); - - /* init capture fps according to output */ - self->v4l2capture->info.fps_d = self->v4l2output->info.fps_d; - self->v4l2capture->info.fps_n = self->v4l2output->info.fps_n; - - /* For decoders G_FMT returns coded size, G_SELECTION returns visible size - * in the compose rectangle. gst_v4l2_object_acquire_format() checks both - * and returns the visible size as with/height and the coded size as - * padding. */ - if (!gst_v4l2_object_acquire_format (self->v4l2capture, &info)) - goto not_negotiated; - - /* gst_v4l2_object_acquire_format() does not set fps, copy from sink */ - info.fps_n = self->v4l2output->info.fps_n; - info.fps_d = self->v4l2output->info.fps_d; - - /* Create caps from the acquired format, remove the format field */ - acquired_caps = gst_video_info_to_caps (&info); - GST_DEBUG_OBJECT (self, "Acquired caps: %" GST_PTR_FORMAT, acquired_caps); - st = gst_caps_get_structure (acquired_caps, 0); - gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", - NULL); - - /* Probe currently available pixel formats */ - available_caps = gst_caps_copy (self->probed_srccaps); - GST_DEBUG_OBJECT (self, "Available caps: %" GST_PTR_FORMAT, available_caps); - - /* Replace coded size with visible size, we want to negotiate visible size - * with downstream, not coded size. */ - gst_caps_map_in_place (available_caps, gst_v4l2_video_remove_padding, self); - - filter = gst_caps_intersect_full (available_caps, acquired_caps, - GST_CAPS_INTERSECT_FIRST); - GST_DEBUG_OBJECT (self, "Filtered caps: %" GST_PTR_FORMAT, filter); - gst_caps_unref (acquired_caps); - gst_caps_unref (available_caps); - caps = gst_pad_peer_query_caps (decoder->srcpad, filter); - gst_caps_unref (filter); - - GST_DEBUG_OBJECT (self, "Possible decoded caps: %" GST_PTR_FORMAT, caps); - if (gst_caps_is_empty (caps)) { - gst_caps_unref (caps); - goto not_negotiated; - } - - /* Fixate pixel format */ - caps = gst_caps_fixate (caps); - - GST_DEBUG_OBJECT (self, "Chosen decoded caps: %" GST_PTR_FORMAT, caps); - - /* Try to set negotiated format, on success replace acquired format */ - if (gst_v4l2_object_set_format (self->v4l2capture, caps, &error)) - gst_video_info_from_caps (&info, caps); - else - gst_v4l2_clear_error (&error); - gst_caps_unref (caps); - - output_state = gst_video_decoder_set_output_state (decoder, - info.finfo->format, info.width, info.height, self->input_state); - - /* Copy the rest of the information, there might be more in the future */ - output_state->info.interlace_mode = info.interlace_mode; - gst_video_codec_state_unref (output_state); - - if (!gst_video_decoder_negotiate (decoder)) { - if (GST_PAD_IS_FLUSHING (decoder->srcpad)) - goto flushing; - else - goto not_negotiated; - } - - /* Ensure our internal pool is activated */ - if (!gst_buffer_pool_set_active (GST_BUFFER_POOL (self->v4l2capture->pool), - TRUE)) - goto activate_failed; } task_state = gst_pad_get_task_state (GST_VIDEO_DECODER_SRC_PAD (self)); -- GitLab From 24eb35f113404f8f5bd8f2a69bb14058441cb656 Mon Sep 17 00:00:00 2001 From: Ming Qian Date: Tue, 19 Oct 2021 16:10:06 +0800 Subject: [PATCH 2/2] v4l2videodec : enable resolution change The dynamic resolution changes when the sequence starts when the decoder detects a coded frame with one or more of the following parameters different from those previously established (and reflected by corresponding queries): 1.coded resolution (OUTPUT width and height), 2.visible resolution (selection rectangles), 3.the minimum number of buffers needed for decoding, 4.bit-depth of the bitstream has been changed. Although gstreamer parser has parsed the stream resolution. but there are some case that we need to handle resolution change event. 1. bit-depth is different from the negotiated format. 2. the capture buffer count can meet the demand 3. there are some hardware limitations that the decoded resolution may be larger than the display size. For example, the stream size is 1920x1080, but some vpu may decode it to 1920x1088. Part-of: --- .../sys/v4l2/gstv4l2videodec.c | 36 ++++++++++++++++--- .../sys/v4l2/gstv4l2videodec.h | 3 ++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c index c2aa0d114d6..674e3c1318e 100644 --- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c +++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c @@ -185,6 +185,7 @@ gst_v4l2_video_dec_start (GstVideoDecoder * decoder) gst_v4l2_object_unlock (self->v4l2output); g_atomic_int_set (&self->active, TRUE); + g_atomic_int_set (&self->capture_configuration_change, FALSE); self->output_flow = GST_FLOW_OK; return TRUE; @@ -609,6 +610,8 @@ gst_v4l2_video_dec_setup_capture (GstVideoDecoder * decoder) /* Copy the rest of the information, there might be more in the future */ output_state->info.interlace_mode = info.interlace_mode; gst_video_codec_state_unref (output_state); + gst_v4l2_buffer_pool_enable_resolution_change (GST_V4L2_BUFFER_POOL + (self->v4l2capture->pool)); if (!gst_video_decoder_negotiate (decoder)) { if (GST_PAD_IS_FLUSHING (decoder->srcpad)) @@ -642,15 +645,30 @@ static void gst_v4l2_video_dec_loop (GstVideoDecoder * decoder) { GstV4l2VideoDec *self = GST_V4L2_VIDEO_DEC (decoder); - GstV4l2BufferPool *v4l2_pool = GST_V4L2_BUFFER_POOL (self->v4l2capture->pool); + GstV4l2BufferPool *v4l2_pool; GstBufferPool *pool; GstVideoCodecFrame *frame; GstBuffer *buffer = NULL; GstFlowReturn ret; + GST_VIDEO_DECODER_STREAM_LOCK (decoder); + if (g_atomic_int_get (&self->capture_configuration_change)) { + gst_v4l2_object_stop (self->v4l2capture); + ret = gst_v4l2_video_dec_setup_capture (decoder); + if (ret != GST_FLOW_OK) + return; + g_atomic_int_set (&self->capture_configuration_change, FALSE); + } + GST_VIDEO_DECODER_STREAM_UNLOCK (decoder); + + if (G_UNLIKELY (!GST_V4L2_IS_ACTIVE (self->v4l2capture))) + return; + GST_LOG_OBJECT (decoder, "Allocate output buffer"); + v4l2_pool = GST_V4L2_BUFFER_POOL (self->v4l2capture->pool); self->output_flow = GST_FLOW_OK; + do { /* We cannot use the base class allotate helper since it taking the internal * stream lock. we know that the acquire may need to poll until more frames @@ -667,6 +685,12 @@ gst_v4l2_video_dec_loop (GstVideoDecoder * decoder) ret = gst_buffer_pool_acquire_buffer (pool, &buffer, NULL); g_object_unref (pool); + if (ret == GST_V4L2_FLOW_RESOLUTION_CHANGE) { + GST_WARNING_OBJECT (decoder, "Received resolution change"); + g_atomic_int_set (&self->capture_configuration_change, TRUE); + return; + } + if (ret != GST_FLOW_OK) goto beach; @@ -754,10 +778,12 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder, goto not_negotiated; } - ret = gst_v4l2_video_dec_setup_capture (decoder); - if (ret != GST_FLOW_OK) { - GST_ERROR_OBJECT (decoder, "setup capture fail\n"); - goto not_negotiated; + if (!g_atomic_int_get (&self->capture_configuration_change)) { + ret = gst_v4l2_video_dec_setup_capture (decoder); + if (ret != GST_FLOW_OK) { + GST_ERROR_OBJECT (decoder, "setup capture fail\n"); + goto not_negotiated; + } } if (G_UNLIKELY (!gst_buffer_pool_is_active (pool))) { diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.h b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.h index 6696fcb35e9..5af08ac9354 100644 --- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.h +++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.h @@ -62,6 +62,9 @@ struct _GstV4l2VideoDec GstVideoCodecState *input_state; gboolean active; GstFlowReturn output_flow; + + /* dynamic resolution change flag */ + gboolean capture_configuration_change; }; struct _GstV4l2VideoDecClass -- GitLab