Problem on executing WebRTC pipeline on Android
We are trying to port an existing WerbRTC pipeline used for Audio/Video call to Android, the application is based on Qt. and the pipeline is as follows:
- we get prepare the pipeline for audio:
GstElement *source = gst_element_factory_make("autoaudiosrc", nullptr);
GstElement *volume = gst_element_factory_make("volume", "srclevel");
GstElement *convert = gst_element_factory_make("audioconvert", nullptr);
GstElement *resample = gst_element_factory_make("audioresample", nullptr);
GstElement *queue1 = gst_element_factory_make("queue", nullptr);
GstElement *opusenc = gst_element_factory_make("opusenc", nullptr);
GstElement *rtp = gst_element_factory_make("rtpopuspay", nullptr);
GstElement *queue2 = gst_element_factory_make("queue", nullptr);
GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr);
GstCaps *rtpcaps = gst_caps_new_simple("application/x-rtp",
"media", G_TYPE_STRING, "audio",
"encoding-name", G_TYPE_STRING, "OPUS",
"payload", G_TYPE_INT, opusPayloadType,
nullptr);
g_object_set(capsfilter, "caps", rtpcaps, nullptr);
gst_caps_unref(rtpcaps);
GstElement *webrtcbin = gst_element_factory_make("webrtcbin", "webrtcbin");
g_object_set(webrtcbin, "bundle-policy", GST_WEBRTC_BUNDLE_POLICY_MAX_BUNDLE, nullptr);
pipe_ = gst_pipeline_new(nullptr);
gst_bin_add_many(GST_BIN(pipe_), source, volume, convert, resample,
queue1, opusenc, rtp, queue2, capsfilter, webrtcbin, nullptr);
gst_element_link_many(source, volume, convert, resample, queue1,
opusenc, rtp, queue2, capsfilter, webrtcbin, nullptr);
- we also try to attach the local camera stream to pipeline using:
GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr);
GstElement *tee = gst_element_factory_make("tee", "videosrctee");
gst_bin_add_many(GST_BIN(pipe_), videoconvert, tee, nullptr);
std::pair<int, int> resolution ={ 640, 480 };
std::pair<int, int> frameRate ={ 30, 1 };
GstElement *camera = gst_element_factory_make("ahcsrc", nullptr);
GstCaps *caps = gst_caps_new_simple("video/x-raw",
"width", G_TYPE_INT, resolution.first,
"height", G_TYPE_INT, resolution.second,
"framerate", GST_TYPE_FRACTION, frameRate.first, frameRate.second,
nullptr);
camerafilter = gst_element_factory_make("capsfilter", "camerafilter");
g_object_set(camerafilter, "caps", caps, nullptr);
gst_caps_unref(caps);
gst_bin_add_many(GST_BIN(pipe_), camera, camerafilter, nullptr);
if (!gst_element_link_many(camera, videoconvert, camerafilter, nullptr)) {
return false;
}
if (callType_ == CallType::VIDEO && !gst_element_link(camerafilter, tee)) {
return false;
}
- then we attached the video stream pipeline
GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr);
g_object_set(vp8enc, "deadline", 1, nullptr);
g_object_set(vp8enc, "error-resilient", 1, nullptr);
GstElement *rtpvp8pay = gst_element_factory_make("rtpvp8pay", nullptr);
GstElement *rtpqueue = gst_element_factory_make("queue", nullptr);
GstElement *rtpcapsfilter = gst_element_factory_make("capsfilter", nullptr);
GstCaps *rtpcaps = gst_caps_new_simple("application/x-rtp",
"media", G_TYPE_STRING, "video",
"encoding-name", G_TYPE_STRING, "VP8",
"payload", G_TYPE_INT, vp8PayloadType,
nullptr);
g_object_set(rtpcapsfilter, "caps", rtpcaps, nullptr);
gst_caps_unref(rtpcaps);
gst_bin_add_many(GST_BIN(pipe_), queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, nullptr);
GstElement *webrtcbin = gst_bin_get_by_name(GST_BIN(pipe_), "webrtcbin");
if (!gst_element_link_many(tee, queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, webrtcbin, nullptr)) {
gst_object_unref(webrtcbin);
return false;
}
gst_object_unref(webrtcbin);
- finally we pass the pipeline to a
_videoItem
ofGstGLVideoItem
type:
GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *compositor = gst_element_factory_make("compositor", "compositor");
GstElement *glupload = gst_element_factory_make("glupload", nullptr);
GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr);
GstElement *qmlglsink = gst_element_factory_make("qmlglsink", nullptr);
GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr);
g_object_set(compositor, "background", 1, nullptr);
g_object_set(qmlglsink, "widget", _videoItem, nullptr);
g_object_set(glsinkbin, "sink", qmlglsink, nullptr);
gst_bin_add_many(GST_BIN(pipe), queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr);
gst_element_link_many(queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr);
gst_element_sync_state_with_parent(queue);
gst_element_sync_state_with_parent(compositor);
gst_element_sync_state_with_parent(glupload);
gst_element_sync_state_with_parent(glcolorconvert);
gst_element_sync_state_with_parent(glsinkbin);
following the mentioned steps, the camera usage indicator turns on on the Android device, but we don't receive any stream on either side, and on Mobile we just receive a black screen
I also attached the application logs including the GStreamer logs.