xrt: Add multi client and overlay client support

src/xrt/compositor/client/comp_gl_client.c

@@ -92,6 +92,17 @@ client_gl_swapchain_release_image(struct xrt_swapchain *xsc, uint32_t index)
  *
  */
 
+static xrt_result_t
+client_gl_compositor_prepare_session(struct xrt_compositor *xc,
+                                     struct xrt_session_prepare_info *xspi)
+{
+	struct client_gl_compositor *c = client_gl_compositor(xc);
+
+	// Pipe down call into fd compositor.
+	return xrt_comp_prepare_session(&c->xcfd->base, xspi);
+}
+
+
 static xrt_result_t
 client_gl_compositor_begin_session(struct xrt_compositor *xc,
                                    enum xrt_view_type type)
@@ -316,6 +327,16 @@ client_gl_swapchain_create(struct xrt_compositor *xc,
 	return &sc->base.base;
 }
 
+static xrt_result_t
+client_gl_compositor_poll_events(struct xrt_compositor *xc,
+                                 union xrt_compositor_event *out_xce)
+{
+	struct client_gl_compositor *c = client_gl_compositor(xc);
+
+	// Pipe down call into fd compositor.
+	return xrt_comp_poll_events(&c->xcfd->base, out_xce);
+}
+
 static void
 client_gl_compositor_destroy(struct xrt_compositor *xc)
 {
@@ -328,6 +349,7 @@ client_gl_compositor_init(struct client_gl_compositor *c,
                           client_gl_get_procaddr get_gl_procaddr)
 {
 	c->base.base.create_swapchain = client_gl_swapchain_create;
+	c->base.base.prepare_session = client_gl_compositor_prepare_session;
 	c->base.base.begin_session = client_gl_compositor_begin_session;
 	c->base.base.end_session = client_gl_compositor_end_session;
 	c->base.base.wait_frame = client_gl_compositor_wait_frame;
@@ -339,6 +361,7 @@ client_gl_compositor_init(struct client_gl_compositor *c,
 	c->base.base.layer_quad = client_gl_compositor_layer_quad;
 	c->base.base.layer_commit = client_gl_compositor_layer_commit;
 	c->base.base.destroy = client_gl_compositor_destroy;
+	c->base.base.poll_events = client_gl_compositor_poll_events;
 	c->xcfd = xcfd;
 
 	// Passthrough our formats from the fd compositor to the client.

src/xrt/compositor/client/comp_vk_client.c

@@ -140,6 +140,16 @@ client_vk_swapchain_release_image(struct xrt_swapchain *xsc, uint32_t index)
  *
  */
 
+static xrt_result_t
+client_vk_compositor_poll_events(struct xrt_compositor *xc,
+                                 union xrt_compositor_event *out_xce)
+{
+	struct client_vk_compositor *c = client_vk_compositor(xc);
+
+	// Pipe down call into fd compositor.
+	return xrt_comp_poll_events(&c->xcfd->base, out_xce);
+}
+
 static void
 client_vk_compositor_destroy(struct xrt_compositor *xc)
 {
@@ -159,6 +169,16 @@ client_vk_compositor_destroy(struct xrt_compositor *xc)
 	free(c);
 }
 
+static xrt_result_t
+client_vk_compositor_prepare_session(struct xrt_compositor *xc,
+                                     struct xrt_session_prepare_info *xspi)
+{
+	struct client_vk_compositor *c = client_vk_compositor(xc);
+
+	// Pipe down call into fd compositor.
+	return xrt_comp_prepare_session(&c->xcfd->base, xspi);
+}
+
 static xrt_result_t
 client_vk_compositor_begin_session(struct xrt_compositor *xc,
                                    enum xrt_view_type type)
@@ -444,6 +464,7 @@ client_vk_compositor_create(struct xrt_compositor_fd *xcfd,
 	    U_TYPED_CALLOC(struct client_vk_compositor);
 
 	c->base.base.create_swapchain = client_vk_swapchain_create;
+	c->base.base.prepare_session = client_vk_compositor_prepare_session;
 	c->base.base.begin_session = client_vk_compositor_begin_session;
 	c->base.base.end_session = client_vk_compositor_end_session;
 	c->base.base.wait_frame = client_vk_compositor_wait_frame;
@@ -455,6 +476,8 @@ client_vk_compositor_create(struct xrt_compositor_fd *xcfd,
 	c->base.base.layer_quad = client_vk_compositor_layer_quad;
 	c->base.base.layer_commit = client_vk_compositor_layer_commit;
 	c->base.base.destroy = client_vk_compositor_destroy;
+	c->base.base.poll_events = client_vk_compositor_poll_events;
+
 	c->xcfd = xcfd;
 	// passthrough our formats from the fd compositor to the client
 	for (uint32_t i = 0; i < xcfd->base.num_formats; i++) {

src/xrt/include/xrt/xrt_compositor.h

@@ -347,6 +347,61 @@ xrt_swapchain_destroy(struct xrt_swapchain **xsc_ptr)
 	*xsc_ptr = NULL;
 }
 
+enum xrt_compositor_event_type
+{
+	XRT_COMPOSITOR_EVENT_NONE = 0,
+	XRT_COMPOSITOR_EVENT_STATE_CHANGE = 1,
+	XRT_COMPOSITOR_EVENT_OVERLAY_CHANGE = 2,
+};
+
+/*!
+ * Session state changes event.
+ *
+ * @ingroup xrt_iface
+ */
+struct xrt_compositor_event_state_change
+{
+	enum xrt_compositor_event_type type;
+	bool visible;
+	bool focused;
+};
+
+/*!
+ * Primary session state changes event.
+ *
+ * @ingroup xrt_iface
+ */
+struct xrt_compositor_event_overlay
+{
+	enum xrt_compositor_event_type type;
+	bool primary_focused;
+};
+
+/*!
+ * Compositor events union.
+ *
+ * @ingroup xrt_iface
+ */
+union xrt_compositor_event {
+	enum xrt_compositor_event_type type;
+	struct xrt_compositor_event_state_change state;
+	struct xrt_compositor_event_state_change overlay;
+};
+
+
+
+/*!
+ * Session prepare information, mostly overlay extension data.
+ *
+ * @ingroup xrt_iface
+ */
+struct xrt_session_prepare_info
+{
+	bool is_overlay;
+	uint64_t flags;
+	uint32_t z_order;
+};
+
 /*!
  * @interface xrt_compositor
  *
@@ -386,12 +441,14 @@ struct xrt_compositor
 	 *
 	 * This function is very much WIP.
 	 */
-	xrt_result_t (*poll_events)(struct xrt_compositor *xc, uint64_t *WIP);
+	xrt_result_t (*poll_events)(struct xrt_compositor *xc,
+	                            union xrt_compositor_event *out_xce);
 
 	/*!
 	 * This function is implicit in the OpenXR spec but made explicit here.
 	 */
-	xrt_result_t (*prepare_session)(struct xrt_compositor *xc);
+	xrt_result_t (*prepare_session)(struct xrt_compositor *xc,
+	                                struct xrt_session_prepare_info *xspi);
 
 	/*!
 	 * See xrBeginSession.
@@ -524,9 +581,10 @@ xrt_comp_create_swapchain(struct xrt_compositor *xc,
  * @public @memberof xrt_compositor
  */
 static inline xrt_result_t
-xrt_comp_poll_events(struct xrt_compositor *xc, uint64_t *WIP)
+xrt_comp_poll_events(struct xrt_compositor *xc,
+                     union xrt_compositor_event *out_xce)
 {
-	return xc->poll_events(xc, WIP);
+	return xc->poll_events(xc, out_xce);
 }
 
 /*!
@@ -537,9 +595,10 @@ xrt_comp_poll_events(struct xrt_compositor *xc, uint64_t *WIP)
  * @public @memberof xrt_compositor
  */
 static inline xrt_result_t
-xrt_comp_prepare_session(struct xrt_compositor *xc)
+xrt_comp_prepare_session(struct xrt_compositor *xc,
+                         struct xrt_session_prepare_info *xspi)
 {
-	return xc->prepare_session(xc);
+	return xc->prepare_session(xc, xspi);
 }
 
 /*!

src/xrt/include/xrt/xrt_instance.h

@@ -10,6 +10,8 @@
 #pragma once
 
 #include "xrt/xrt_compiler.h"
+#include "xrt/xrt_defines.h"
+
 
 #ifdef __cplusplus
 extern "C" {
@@ -20,11 +22,22 @@ struct xrt_prober;
 struct xrt_device;
 struct xrt_compositor_fd;
 
+
 /*!
  * @ingroup xrt_iface
  * @{
  */
 
+#define XRT_MAX_APPLICATION_NAME_SIZE 128
+
+/*!
+ * Information provided by the application at instance create time.
+ */
+struct xrt_instance_info
+{
+	char application_name[XRT_MAX_APPLICATION_NAME_SIZE];
+};
+
 /*!
  * @interface xrt_instance
  *
@@ -131,6 +144,7 @@ struct xrt_instance
 	/*!
 	 * @}
 	 */
+	struct xrt_instance_info instance_info;
 };
 
 /*!
@@ -220,7 +234,8 @@ xrt_instance_destroy(struct xrt_instance **xinst_ptr)
  * @relates xrt_instance
  */
 int
-xrt_instance_create(struct xrt_instance **out_xinst);
+xrt_instance_create(struct xrt_instance **out_xinst,
+                    struct xrt_instance_info *ii);
 
 /*!
  * @}

src/xrt/ipc/CMakeLists.txt

@@ -38,6 +38,7 @@ add_library(ipc_client STATIC
 	)
 target_include_directories(ipc_client INTERFACE
 	${CMAKE_CURRENT_SOURCE_DIR}
+	${CMAKE_CURRENT_BINARY_DIR}
 	)
 target_include_directories(ipc_client PRIVATE
 	${CMAKE_CURRENT_SOURCE_DIR}
@@ -65,6 +66,7 @@ add_library(ipc_server STATIC
 target_include_directories(ipc_server
 	INTERFACE
 	${CMAKE_CURRENT_SOURCE_DIR}
+	${CMAKE_CURRENT_BINARY_DIR}
 	)
 target_include_directories(ipc_server PRIVATE
 	${CMAKE_CURRENT_SOURCE_DIR}/../compositor

src/xrt/ipc/ipc_client_compositor.c

@@ -226,6 +226,31 @@ ipc_compositor_swapchain_create(struct xrt_compositor *xc,
 	return &ics->base.base;
 }
 
+static xrt_result_t
+ipc_compositor_prepare_session(struct xrt_compositor *xc,
+                               struct xrt_session_prepare_info *xspi)
+{
+	struct ipc_client_compositor *icc = ipc_client_compositor(xc);
+
+	IPC_SPEW(icc->ipc_c, "IPC: compositor create session");
+
+	IPC_CALL_CHK(ipc_call_session_create(icc->ipc_c, xspi));
+	return res;
+}
+
+static xrt_result_t
+ipc_compositor_poll_events(struct xrt_compositor *xc,
+                           union xrt_compositor_event *out_xce)
+{
+	struct ipc_client_compositor *icc = ipc_client_compositor(xc);
+
+	IPC_SPEW(icc->ipc_c, "IPC: polling for events");
+
+	IPC_CALL_CHK(ipc_call_compositor_poll_events(icc->ipc_c, out_xce));
+
+	return res;
+}
+
 static xrt_result_t
 ipc_compositor_begin_session(struct xrt_compositor *xc,
                              enum xrt_view_type view_type)
@@ -458,6 +483,7 @@ ipc_client_compositor_create(ipc_connection_t *ipc_c,
 	    U_TYPED_CALLOC(struct ipc_client_compositor);
 
 	c->base.base.create_swapchain = ipc_compositor_swapchain_create;
+	c->base.base.prepare_session = ipc_compositor_prepare_session;
 	c->base.base.begin_session = ipc_compositor_begin_session;
 	c->base.base.end_session = ipc_compositor_end_session;
 	c->base.base.wait_frame = ipc_compositor_wait_frame;
@@ -469,6 +495,7 @@ ipc_client_compositor_create(ipc_connection_t *ipc_c,
 	c->base.base.layer_quad = ipc_compositor_layer_quad;
 	c->base.base.layer_commit = ipc_compositor_layer_commit;
 	c->base.base.destroy = ipc_compositor_destroy;
+	c->base.base.poll_events = ipc_compositor_poll_events;
 	c->ipc_c = ipc_c;
 
 	// fetch our format list on client compositor construction

src/xrt/ipc/ipc_client_instance.c

@@ -183,7 +183,8 @@ ipc_client_instance_destroy(struct xrt_instance *xinst)
  * @public @memberof ipc_instance
  */
 int
-ipc_instance_create(struct xrt_instance **out_xinst)
+ipc_instance_create(struct xrt_instance **out_xinst,
+                    struct xrt_instance_info *i_info)
 {
 	struct ipc_client_instance *ii =
 	    U_TYPED_CALLOC(struct ipc_client_instance);
@@ -223,6 +224,17 @@ ipc_instance_create(struct xrt_instance **out_xinst)
 		return -1;
 	}
 
+	struct ipc_app_state desc = {0};
+	desc.info = *i_info;
+	desc.pid = getpid(); // Extra info.
+
+	r = ipc_call_system_set_client_info(&ii->ipc_c, &desc);
+	if (r != XRT_SUCCESS) {
+		IPC_ERROR(&ii->ipc_c, "Failed to set instance info");
+		free(ii);
+		return -1;
+	}
+
 	const int flags = MAP_SHARED;
 	const int access = PROT_READ | PROT_WRITE;
 	const size_t size = sizeof(struct ipc_shared_memory);

src/xrt/ipc/ipc_protocol.h

@@ -4,16 +4,20 @@
  * @file
  * @brief  Common protocol definition.
  * @author Pete Black <pblack@collabora.com>
+ * @author Jakob Bornecrantz <jakob@collabora.com>
  * @ingroup ipc
  */
 
 #pragma once
 
-#include "xrt/xrt_tracking.h"
-#include "xrt/xrt_device.h"
 #include "xrt/xrt_compiler.h"
 #include "xrt/xrt_compositor.h"
 #include "xrt/xrt_results.h"
+#include "xrt/xrt_defines.h"
+#include "xrt/xrt_instance.h"
+#include "xrt/xrt_compositor.h"
+#include "xrt/xrt_device.h"
+#include "xrt/xrt_tracking.h"
 
 
 #define IPC_MSG_SOCK_FILE "/tmp/monado_comp_ipc"
@@ -24,7 +28,9 @@
 #define IPC_MAX_FORMATS 32 // max formats our server-side compositor supports
 #define IPC_MAX_DEVICES 8  // max number of devices we will map via shared mem
 #define IPC_MAX_LAYERS 16
-#define IPC_MAX_SLOTS 3
+#define IPC_MAX_SLOTS 128
+#define IPC_MAX_CLIENTS 8
+#define IPC_EVENT_QUEUE_SIZE 32
 
 #define IPC_SHARED_MAX_DEVICES 8
 #define IPC_SHARED_MAX_INPUTS 1024
@@ -163,10 +169,24 @@ struct ipc_shared_memory
 	struct ipc_layer_slot slots[IPC_MAX_SLOTS];
 };
 
-/*
- *
- * Rest of protocol is generated.
+struct ipc_client_list
+{
+	int32_t ids[IPC_MAX_CLIENTS];
+};
+
+/*!
+ * State for a connected application.
  *
+ * @ingroup ipc
  */
-
-#include "ipc_protocol_generated.h"
+struct ipc_app_state
+{
+	bool primary_application;
+	bool session_active;
+	bool session_visible;
+	bool session_focused;
+	bool session_overlay;
+	uint32_t z_order;
+	pid_t pid;
+	struct xrt_instance_info info;
+};

src/xrt/ipc/ipc_server.h

@@ -64,7 +64,7 @@ extern "C" {
 
 #define IPC_SERVER_NUM_XDEVS 8
 #define IPC_MAX_CLIENT_SWAPCHAINS 32
-#define IPC_MAX_CLIENTS 8
+//#define IPC_MAX_CLIENTS 8
 
 struct xrt_instance;
 struct xrt_compositor;
@@ -86,6 +86,14 @@ struct ipc_swapchain_data
 	bool active;
 };
 
+
+struct ipc_queued_event
+{
+	bool pending;
+	uint64_t timestamp;
+	union xrt_compositor_event event;
+};
+
 /*!
  * Holds the state for a single client.
  *
@@ -117,7 +125,28 @@ struct ipc_client_state
 	//! Whether we are currently rendering @ref render_state
 	bool rendering_state;
 
-	bool active;
+	//! The frame timing state.
+	struct u_rt_helper urth;
+
+	struct ipc_app_state client_state;
+	struct ipc_queued_event queued_events[IPC_EVENT_QUEUE_SIZE];
+
+	int server_thread_index;
+};
+
+enum ipc_thread_state
+{
+	IPC_THREAD_READY,
+	IPC_THREAD_STARTING,
+	IPC_THREAD_RUNNING,
+	IPC_THREAD_STOPPING,
+};
+
+struct ipc_thread
+{
+	struct os_thread thread;
+	volatile enum ipc_thread_state state;
+	volatile struct ipc_client_state ics;
 };
 
 /*!
@@ -159,14 +188,13 @@ struct ipc_server
 	bool print_debug;
 	bool print_spew;
 
-	// Hack for now.
-	struct os_thread thread;
-	volatile bool thread_started;
-	volatile bool thread_stopping;
-	volatile struct ipc_client_state thread_state;
+	struct ipc_thread threads[IPC_MAX_CLIENTS];
 
+	volatile uint32_t current_slot_index;
 
-	struct u_rt_helper urth;
+	int active_client_index;
+	int last_active_client_index;
+	struct os_mutex global_state_lock;
 };
 
 /*!
@@ -177,6 +205,14 @@ struct ipc_server
 int
 ipc_server_main(int argc, char **argv);
 
+/*!
+ * Called by client threads to manage global state
+ *
+ * @ingroup ipc_server
+ */
+void
+update_server_state(struct ipc_server *vs);
+
 /*!
  * Thread function for the client side dispatching.
  *

src/xrt/ipc/ipc_server_client.c

@@ -48,97 +48,231 @@ ipc_handle_instance_get_shm_fd(volatile struct ipc_client_state *cs,
 	*out_num_fds = 1;
 	return XRT_SUCCESS;
 }
+xrt_result_t
+ipc_handle_session_create(volatile struct ipc_client_state *ics,
+                          struct xrt_session_prepare_info *xspi)
+{
+	ics->client_state.session_active = false;
+	ics->client_state.session_overlay = false;
+	ics->client_state.session_visible = false;
+
+	if (xspi->is_overlay) {
+		ics->client_state.session_overlay = true;
+		ics->client_state.z_order = xspi->z_order;
+	}
+
+	update_server_state(ics->server);
+	return XRT_SUCCESS;
+}
 
 xrt_result_t
-ipc_handle_session_begin(volatile struct ipc_client_state *cs)
+ipc_handle_session_begin(volatile struct ipc_client_state *ics)
 {
-	cs->active = true;
+	// ics->client_state.session_active = true;
+	// update_server_state(ics->server);
 	return XRT_SUCCESS;
 }
 
 xrt_result_t
-ipc_handle_session_end(volatile struct ipc_client_state *cs)
+ipc_handle_session_end(volatile struct ipc_client_state *ics)
 {
-	cs->active = false;
+	ics->client_state.session_active = false;
+	update_server_state(ics->server);
 	return XRT_SUCCESS;
 }
 
 xrt_result_t
-ipc_handle_compositor_get_formats(volatile struct ipc_client_state *cs,
+ipc_handle_compositor_get_formats(volatile struct ipc_client_state *ics,
                                   struct ipc_formats_info *out_info)
 {
-	out_info->num_formats = cs->xc->num_formats;
-	for (size_t i = 0; i < cs->xc->num_formats; i++) {
-		out_info->formats[i] = cs->xc->formats[i];
+	out_info->num_formats = ics->xc->num_formats;
+	for (size_t i = 0; i < ics->xc->num_formats; i++) {
+		out_info->formats[i] = ics->xc->formats[i];
 	}
 
 	return XRT_SUCCESS;
 }
 
 xrt_result_t
-ipc_handle_compositor_wait_frame(volatile struct ipc_client_state *cs,
+ipc_handle_compositor_wait_frame(volatile struct ipc_client_state *ics,
                                  int64_t *out_frame_id,
                                  uint64_t *predicted_display_time,
                                  uint64_t *wake_up_time,
                                  uint64_t *predicted_display_period,
                                  uint64_t *min_display_period)
 {
-	u_rt_helper_predict(&cs->server->urth, out_frame_id,
+	os_mutex_lock(&ics->server->global_state_lock);
+
+	u_rt_helper_predict((struct u_rt_helper *)&ics->urth, out_frame_id,
 	                    predicted_display_time, wake_up_time,
 	                    predicted_display_period, min_display_period);
 
+	os_mutex_unlock(&ics->server->global_state_lock);
+
+	ics->client_state.session_active = true;
+	update_server_state(ics->server);
+
 	return XRT_SUCCESS;
 }
 
 xrt_result_t
-ipc_handle_compositor_wait_woke(volatile struct ipc_client_state *cs,
+ipc_handle_compositor_wait_woke(volatile struct ipc_client_state *ics,
                                 int64_t frame_id)
 {
-	u_rt_helper_mark_wait_woke(&cs->server->urth, frame_id);
+	os_mutex_lock(&ics->server->global_state_lock);
+
+	u_rt_helper_mark_wait_woke((struct u_rt_helper *)&ics->urth, frame_id);
+
+	os_mutex_unlock(&ics->server->global_state_lock);
 
 	return XRT_SUCCESS;
 }
 
 xrt_result_t
-ipc_handle_compositor_begin_frame(volatile struct ipc_client_state *cs,
+ipc_handle_compositor_begin_frame(volatile struct ipc_client_state *ics,
                                   int64_t frame_id)
 {
-	u_rt_helper_mark_begin(&cs->server->urth, frame_id);
+	os_mutex_lock(&ics->server->global_state_lock);
+
+	u_rt_helper_mark_begin((struct u_rt_helper *)&ics->urth, frame_id);
+
+	os_mutex_unlock(&ics->server->global_state_lock);
 
 	return XRT_SUCCESS;
 }
 
 xrt_result_t
-ipc_handle_compositor_discard_frame(volatile struct ipc_client_state *cs,
+ipc_handle_compositor_discard_frame(volatile struct ipc_client_state *ics,
                                     int64_t frame_id)
 {
-	u_rt_helper_mark_discarded(&cs->server->urth, frame_id);
+	os_mutex_lock(&ics->server->global_state_lock);
+
+	u_rt_helper_mark_discarded((struct u_rt_helper *)&ics->urth, frame_id);
+
+	os_mutex_unlock(&ics->server->global_state_lock);

[Ryan Pavlik]

Can you document the instance info parameter here in the doxygen comment above? It's causing a build warning.

[Jakob Bornecrantz]

On it!