Find some issues about VNC backend + gl renderer, hope helpful for weston maintainers to fix them

I have made weston 13.0.0 VNC backend + gl renderer work on conatainer with Nvidia Driver. But I have found some bugs/issues and fixed them. Below is my findings and temp fixes, Hope they are helpful for weston maintainers to give a formal fix! thanks:

==============================================================================================================================================
From fc96bb0ee537cd573a90ca2f6531a535086085d8 Mon Sep 17 00:00:00 2001
From: Wu Zhongmin <zhongmin.wzm@antgroup.com>
Date: Wed, 13 Mar 2024 08:17:57 +0000
Subject: [PATCH] vnc: add dmabuf support for vnc backend

Now vnc backend did not support dmabuf. it can work normally with gl renderer on Nvidia EGL Wayland Environment

Also fix some bugs in gl-renderer with vnc backend:
1. pixman_image_create_bits_no_clear use the wrong value of stride, it should be in bytes not pixels
2. GL_PACK_ROW_LENGTH shoud be reset from stride (fb's width),  since the exactly tmp buffer with the need size is malloced.
3. the fbo should be recovered when finish repaint, otherwise x11 output would be black screen when x11,vnc backends work together
---
 libweston/backend-vnc/vnc.c         | 7 +++++++
 libweston/renderer-gl/gl-renderer.c | 7 ++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/libweston/backend-vnc/vnc.c b/libweston/backend-vnc/vnc.c
index 9fcf7e30..53103e21 100644
--- a/libweston/backend-vnc/vnc.c
+++ b/libweston/backend-vnc/vnc.c
@@ -54,6 +54,7 @@
 #include <libweston/backend-vnc.h>
 #include <libweston/weston-log.h>
 #include "pixel-formats.h"
+#include "linux-dmabuf.h"
 #include "pixman-renderer.h"
 #include "renderer-gl/gl-renderer.h"
 #include "shared/weston-egl-ext.h"
@@ -1189,6 +1190,12 @@ vnc_backend_create(struct weston_compositor *compositor,
 			weston_log("Unsupported renderer requested\n");
 			goto err_compositor;
 		}
+		if (compositor->renderer->import_dmabuf) {
+			if (linux_dmabuf_setup(compositor) < 0) {
+				weston_log("Error: dmabuf protocol setup failed.\n");
+				goto err_compositor;
+			}
+		}
 	}
 
 	vnc_head_create(backend, "vnc");
diff --git a/libweston/renderer-gl/gl-renderer.c b/libweston/renderer-gl/gl-renderer.c
index ccb41e66..41fbcda7 100644
--- a/libweston/renderer-gl/gl-renderer.c
+++ b/libweston/renderer-gl/gl-renderer.c
@@ -818,6 +818,7 @@ gl_renderer_do_read_pixels(struct gl_renderer *gr,
 			return false;
 
 		read_target = pixman_image_get_data(tmp);
+		glPixelStorei(GL_PACK_ROW_LENGTH, 0);
 	}
 
 	glReadPixels(rect->x, rect->y, rect->width, rect->height,
@@ -1868,6 +1869,9 @@ gl_renderer_repaint_output(struct weston_output *output,
 	const int32_t area_inv_y =
 		go->fb_size.height - go->area.y - go->area.height;
 	struct gl_renderbuffer *rb;
+	GLint saved_fbo = 0;
+	glGetIntegerv(GL_FRAMEBUFFER_BINDING, &saved_fbo);
+
 
 	assert(output->from_blend_to_output_by_backend ||
 	       output->color_outcome->from_blend_to_output == NULL ||
@@ -2044,7 +2048,7 @@ gl_renderer_repaint_output(struct weston_output *output,
 			pixels += extents->x1 - (int)output->pos.c.x;
 		}
 
-		gl_renderer_do_read_pixels(gr, compositor->read_format, pixels, stride, &rect);
+		gl_renderer_do_read_pixels(gr, compositor->read_format, pixels, stride * (compositor->read_format->bpp / 8), &rect);
 
 		if (gr->gl_version >= gr_gl_version(3, 0))
 			glPixelStorei(GL_PACK_ROW_LENGTH, 0);
@@ -2053,6 +2057,7 @@ gl_renderer_repaint_output(struct weston_output *output,
 	pixman_region32_clear(&rb->base.damage);
 
 	gl_renderer_garbage_collect_programs(gr);
+	glBindFramebuffer(GL_FRAMEBUFFER, saved_fbo);
 }
 
 static int
-- 
2.25.1

==============================================================================================================================================

Also Use UltraVNCViewer to test RFB_CLIENT_TO_SERVER_SET_DESKTOP_SIZE event, and found issues:

1. ntohs in "on_client_set_desktop_size_event(struct nvnc_client* client)" is not necessary, well it is not weston's issue. width = ntohs(msg->width); height = ntohs(msg->height);

2. UltraVNCViewer will always send RFB_CLIENT_TO_SERVER_SET_DESKTOP_SIZE every time. but **weston_renderer_resize_output ** will always clear render buffer list even if the size is not changed. While nvnc_fb_pool_resize will not clear fbs if size is changed. Then you will find you can get user data (renderbuffer) from fb, but render buffer list in go is empty... then the caculated damages in repaint_output always 0..

===========================================================================================================================================
diff --git a/libweston/backend-vnc/vnc.c b/libweston/backend-vnc/vnc.c
index 53103e21..869d797d 100644
--- a/libweston/backend-vnc/vnc.c
+++ b/libweston/backend-vnc/vnc.c
@@ -1077,7 +1077,13 @@ vnc_switch_mode(struct weston_output *base, struct weston_mode *target_mode)
 
        weston_renderer_resize_output(base, &fb_size, NULL);
 
-       nvnc_fb_pool_resize(output->fb_pool, target_mode->width,
+       /*nvnc_fb_pool_resize(output->fb_pool, target_mode->width,
+                           target_mode->height, DRM_FORMAT_XRGB8888,
+                           target_mode->width);*/
+       //force remove user data (renderbuffer) in fb pool. since it has been removed
+       //in renderbuffer_list of gl renderer even if size is not changed..
+       nvnc_fb_pool_unref(output->fb_pool);
+       output->fb_pool = nvnc_fb_pool_new(target_mode->width,
                            target_mode->height, DRM_FORMAT_XRGB8888,
                            target_mode->width);