diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 20e10f4cfdc8cebfe21d36dcb9cdcf1ee0a5cb0e..1c3abea1b694463c1fdaf4de8b502862509cb6d3 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -87,6 +87,18 @@ "type": "GstValueArray", "writable": true }, + "overflow": { + "blurb": "Defines how output audio with a longer duration than input text should be handled", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "clip (0)", + "mutable": "ready", + "readable": true, + "type": "GstAwsOverflow", + "writable": true + }, "secret-access-key": { "blurb": "AWS Secret Access Key", "conditionally-available": false, @@ -1372,6 +1384,26 @@ "filename": "gstaws", "license": "MPL", "other-types": { + "GstAwsOverflow": { + "kind": "enum", + "values": [ + { + "desc": "Clip", + "name": "clip", + "value": "0" + }, + { + "desc": "Overlap", + "name": "overlap", + "value": "1" + }, + { + "desc": "Shift", + "name": "shift", + "value": "2" + } + ] + }, "GstAwsPollyEngine": { "kind": "enum", "values": [ @@ -11510,11 +11542,171 @@ "package": "gst-plugin-tracers", "source": "gst-plugin-tracers", "tracers": { - "buffer-lateness": {}, - "pad-push-timings": {}, - "pcap-writer": {}, - "pipeline-snapshot": {}, - "queue-levels": {} + "buffer-lateness": { + "hierarchy": [ + "GstBufferLateness", + "GstTracer", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "signals": { + "write-log": { + "action": true, + "args": [ + { + "name": "arg0", + "type": "gchararray" + } + ], + "return-type": "void", + "when": "last" + } + } + }, + "pad-push-timings": { + "hierarchy": [ + "GstPadPushTimings", + "GstTracer", + "GstObject", + "GInitiallyUnowned", + "GObject" + ] + }, + "pcap-writer": { + "hierarchy": [ + "GstPcapWriter", + "GstTracer", + "GstObject", + "GInitiallyUnowned", + "GObject" + ] + }, + "pipeline-snapshot": { + "hierarchy": [ + "GstPipelineSnapshot", + "GstTracer", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "properties": { + "cleanup-mode": { + "blurb": "Cleanup mode", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "none (2)", + "mutable": "null", + "readable": true, + "type": "GstPipelineSnapshotCleanupMode", + "writable": true + }, + "dot-dir": { + "blurb": "Directory where to place dot files", + "conditionally-available": false, + "construct": false, + "construct-only": true, + "controllable": false, + "default": "NULL", + "mutable": "null", + "readable": true, + "type": "gchararray", + "writable": true + }, + "dot-pipeline-ptr": { + "blurb": "Add pipeline ptr value to dot files", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "false", + "mutable": "null", + "readable": true, + "type": "gboolean", + "writable": true + }, + "dot-prefix": { + "blurb": "Prefix for dot files", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "pipeline-snapshot-", + "mutable": "null", + "readable": true, + "type": "gchararray", + "writable": true + }, + "dot-ts": { + "blurb": "Add timestamp to dot files", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "true", + "mutable": "null", + "readable": true, + "type": "gboolean", + "writable": true + }, + "folder-mode": { + "blurb": "How to create folder each time a snapshot of all pipelines is made", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "none (0)", + "mutable": "null", + "readable": true, + "type": "GstPipelineSnapshotFolderMode", + "writable": true + }, + "xdg-cache": { + "blurb": "Use $XDG_CACHE_DIR/gstreamer-dots", + "conditionally-available": false, + "construct": false, + "construct-only": true, + "controllable": false, + "default": "false", + "mutable": "null", + "readable": true, + "type": "gboolean", + "writable": true + } + }, + "signals": { + "snapshot": { + "action": true, + "args": [], + "return-type": "void", + "when": "last" + } + } + }, + "queue-levels": { + "hierarchy": [ + "GstQueueLevels", + "GstTracer", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "signals": { + "write-log": { + "action": true, + "args": [ + { + "name": "arg0", + "type": "gchararray" + } + ], + "return-type": "void", + "when": "last" + } + } + } }, "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" }, diff --git a/net/aws/src/polly/imp.rs b/net/aws/src/polly/imp.rs index 9a53451e571496d27874d4e2ad830d7e1e0abf8c..af475236526a737ac1a96a1a66e3c04d6ed4ec0e 100644 --- a/net/aws/src/polly/imp.rs +++ b/net/aws/src/polly/imp.rs @@ -21,7 +21,7 @@ use std::sync::Mutex; use std::sync::LazyLock; -use super::{AwsPollyEngine, AwsPollyLanguageCode, AwsPollyVoiceId, CAT}; +use super::{AwsOverflow, AwsPollyEngine, AwsPollyLanguageCode, AwsPollyVoiceId, CAT}; use crate::s3utils::RUNTIME; use anyhow::{anyhow, Error}; @@ -35,6 +35,7 @@ const DEFAULT_ENGINE: AwsPollyEngine = AwsPollyEngine::Neural; const DEFAULT_LANGUAGE_CODE: AwsPollyLanguageCode = AwsPollyLanguageCode::None; const DEFAULT_VOICE_ID: AwsPollyVoiceId = AwsPollyVoiceId::Aria; const DEFAULT_SSML_SET_MAX_DURATION: bool = false; +const DEFAULT_OVERFLOW: AwsOverflow = AwsOverflow::Clip; #[derive(Debug, Clone)] pub(super) struct Settings { @@ -47,6 +48,7 @@ pub(super) struct Settings { voice_id: AwsPollyVoiceId, lexicon_names: gst::Array, ssml_set_max_duration: bool, + overflow: AwsOverflow, } impl Default for Settings { @@ -61,6 +63,7 @@ impl Default for Settings { voice_id: DEFAULT_VOICE_ID, lexicon_names: gst::Array::default(), ssml_set_max_duration: DEFAULT_SSML_SET_MAX_DURATION, + overflow: DEFAULT_OVERFLOW, } } } @@ -158,7 +161,7 @@ impl Polly { } async fn send(&self, inbuf: gst::Buffer) -> Result<gst::Buffer, Error> { - let pts = inbuf + let mut pts = inbuf .pts() .ok_or_else(|| anyhow!("Stream with timestamped buffers required"))?; @@ -227,15 +230,47 @@ impl Polly { })?; let blob = resp.audio_stream.collect().await?; - let mut buf = gst::Buffer::from_slice(blob.into_bytes()); - let mut state = self.state.lock().unwrap(); + let mut bytes = blob.into_bytes(); + + let overflow = self.settings.lock().unwrap().overflow; + + if matches!(overflow, AwsOverflow::Clip) { + let max_expected_bytes = duration + .nseconds() + .mul_div_floor(32_000, 1_000_000_000) + .unwrap() + / 2 + * 2; + + gst::debug!( + CAT, + "Received {} bytes, max expected {}", + bytes.len(), + max_expected_bytes + ); + + bytes.truncate(max_expected_bytes as usize); + } let duration = gst::ClockTime::from_nseconds( - (buf.size() as u64) + (bytes.len() as u64) .mul_div_round(1_000_000_000, 32_000) .unwrap(), ); + let mut buf = gst::Buffer::from_slice(bytes); + let mut state = self.state.lock().unwrap(); + + if let Some(position) = state.out_segment.position() { + if matches!(overflow, AwsOverflow::Shift) && pts < position { + gst::debug!( + CAT, + "received pts {pts} < position {position}, shifting forward" + ); + pts = position; + } + } + let discont = state .out_segment .position() @@ -248,7 +283,7 @@ impl Polly { buf_mut.set_duration(duration); if discont { - gst::log!(CAT, imp = self, "Marking buffer discont"); + gst::debug!(CAT, imp = self, "Marking buffer discont"); buf_mut.set_flags(gst::BufferFlags::DISCONT); } inbuf.foreach_meta(|meta| { @@ -521,6 +556,11 @@ impl ObjectImpl for Polly { .default_value(DEFAULT_SSML_SET_MAX_DURATION) .mutable_ready() .build(), + glib::ParamSpecEnum::builder_with_default("overflow", DEFAULT_OVERFLOW) + .nick("Overflow") + .blurb("Defines how output audio with a longer duration than input text should be handled") + .mutable_ready() + .build(), ] }); @@ -581,6 +621,10 @@ impl ObjectImpl for Polly { let mut settings = self.settings.lock().unwrap(); settings.ssml_set_max_duration = value.get().expect("type checked upstream"); } + "overflow" => { + let mut settings = self.settings.lock().unwrap(); + settings.overflow = value.get::<AwsOverflow>().expect("type checked upstream"); + } _ => unimplemented!(), } } @@ -623,6 +667,10 @@ impl ObjectImpl for Polly { let settings = self.settings.lock().unwrap(); settings.ssml_set_max_duration.to_value() } + "overflow" => { + let settings = self.settings.lock().unwrap(); + settings.overflow.to_value() + } _ => unimplemented!(), } } diff --git a/net/aws/src/polly/mod.rs b/net/aws/src/polly/mod.rs index 945522f1e07e335a7933df4207eccf59b16e46ee..eee0b706b107de154589c62c68246cfe03d0e0a1 100644 --- a/net/aws/src/polly/mod.rs +++ b/net/aws/src/polly/mod.rs @@ -398,6 +398,19 @@ impl From<AwsPollyLanguageCode> for LanguageCode { } } +#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Copy, glib::Enum)] +#[repr(u32)] +#[enum_type(name = "GstAwsOverflow")] +#[non_exhaustive] +pub enum AwsOverflow { + #[enum_value(name = "Clip", nick = "clip")] + Clip = 0, + #[enum_value(name = "Overlap", nick = "overlap")] + Overlap = 1, + #[enum_value(name = "Shift", nick = "shift")] + Shift = 2, +} + glib::wrapper! { pub struct Polly(ObjectSubclass<imp::Polly>) @extends gst::Element, gst::Object; } @@ -408,6 +421,7 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { AwsPollyEngine::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); AwsPollyVoiceId::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); AwsPollyLanguageCode::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); + AwsOverflow::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); } gst::Element::register( Some(plugin),