xrt: Add multi client and overlay client support

src/xrt/compositor/client/comp_gl_client.c

@@ -92,6 +92,17 @@ client_gl_swapchain_release_image(struct xrt_swapchain *xsc, uint32_t index)
  *
  */
 
+static xrt_result_t
+client_gl_compositor_prepare_session(struct xrt_compositor *xc,
+                                     struct xrt_session_prepare_info *xspi)
+{
+	struct client_gl_compositor *c = client_gl_compositor(xc);
+
+	// Pipe down call into fd compositor.
+	return xrt_comp_prepare_session(&c->xcfd->base, xspi);
+}
+
+
 static xrt_result_t
 client_gl_compositor_begin_session(struct xrt_compositor *xc,
                                    enum xrt_view_type type)
@@ -316,6 +327,16 @@ client_gl_swapchain_create(struct xrt_compositor *xc,
 	return &sc->base.base;
 }
 
+static xrt_result_t
+client_gl_compositor_poll_events(struct xrt_compositor *xc,
+                                 union xrt_compositor_event *out_xce)
+{
+	struct client_gl_compositor *c = client_gl_compositor(xc);
+
+	// Pipe down call into fd compositor.
+	return xrt_comp_poll_events(&c->xcfd->base, out_xce);
+}
+
 static void
 client_gl_compositor_destroy(struct xrt_compositor *xc)
 {
@@ -328,6 +349,7 @@ client_gl_compositor_init(struct client_gl_compositor *c,
                           client_gl_get_procaddr get_gl_procaddr)
 {
 	c->base.base.create_swapchain = client_gl_swapchain_create;
+	c->base.base.prepare_session = client_gl_compositor_prepare_session;
 	c->base.base.begin_session = client_gl_compositor_begin_session;
 	c->base.base.end_session = client_gl_compositor_end_session;
 	c->base.base.wait_frame = client_gl_compositor_wait_frame;
@@ -339,6 +361,7 @@ client_gl_compositor_init(struct client_gl_compositor *c,
 	c->base.base.layer_quad = client_gl_compositor_layer_quad;
 	c->base.base.layer_commit = client_gl_compositor_layer_commit;
 	c->base.base.destroy = client_gl_compositor_destroy;
+	c->base.base.poll_events = client_gl_compositor_poll_events;
 	c->xcfd = xcfd;
 
 	// Passthrough our formats from the fd compositor to the client.

src/xrt/compositor/client/comp_vk_client.c

@@ -140,6 +140,16 @@ client_vk_swapchain_release_image(struct xrt_swapchain *xsc, uint32_t index)
  *
  */
 
+static xrt_result_t
+client_vk_compositor_poll_events(struct xrt_compositor *xc,
+                                 union xrt_compositor_event *out_xce)
+{
+	struct client_vk_compositor *c = client_vk_compositor(xc);
+
+	// Pipe down call into fd compositor.
+	return xrt_comp_poll_events(&c->xcfd->base, out_xce);
+}
+
 static void
 client_vk_compositor_destroy(struct xrt_compositor *xc)
 {
@@ -159,6 +169,16 @@ client_vk_compositor_destroy(struct xrt_compositor *xc)
 	free(c);
 }
 
+static xrt_result_t
+client_vk_compositor_prepare_session(struct xrt_compositor *xc,
+                                     struct xrt_session_prepare_info *xspi)
+{
+	struct client_vk_compositor *c = client_vk_compositor(xc);
+
+	// Pipe down call into fd compositor.
+	return xrt_comp_prepare_session(&c->xcfd->base, xspi);
+}
+
 static xrt_result_t
 client_vk_compositor_begin_session(struct xrt_compositor *xc,
                                    enum xrt_view_type type)
@@ -444,6 +464,7 @@ client_vk_compositor_create(struct xrt_compositor_fd *xcfd,
 	    U_TYPED_CALLOC(struct client_vk_compositor);
 
 	c->base.base.create_swapchain = client_vk_swapchain_create;
+	c->base.base.prepare_session = client_vk_compositor_prepare_session;
 	c->base.base.begin_session = client_vk_compositor_begin_session;
 	c->base.base.end_session = client_vk_compositor_end_session;
 	c->base.base.wait_frame = client_vk_compositor_wait_frame;
@@ -455,6 +476,8 @@ client_vk_compositor_create(struct xrt_compositor_fd *xcfd,
 	c->base.base.layer_quad = client_vk_compositor_layer_quad;
 	c->base.base.layer_commit = client_vk_compositor_layer_commit;
 	c->base.base.destroy = client_vk_compositor_destroy;
+	c->base.base.poll_events = client_vk_compositor_poll_events;
+
 	c->xcfd = xcfd;
 	// passthrough our formats from the fd compositor to the client
 	for (uint32_t i = 0; i < xcfd->base.num_formats; i++) {

src/xrt/include/xrt/xrt_compositor.h

@@ -347,6 +347,61 @@ xrt_swapchain_destroy(struct xrt_swapchain **xsc_ptr)
 	*xsc_ptr = NULL;
 }
 
+enum xrt_compositor_event_type
+{
+	XRT_COMPOSITOR_EVENT_NONE = 0,
+	XRT_COMPOSITOR_EVENT_STATE_CHANGE = 1,
+	XRT_COMPOSITOR_EVENT_OVERLAY_CHANGE = 2,
+};
+
+/*!
+ * Session state changes event.
+ *
+ * @ingroup xrt_iface
+ */
+struct xrt_compositor_event_state_change
+{
+	enum xrt_compositor_event_type type;
+	bool visible;
+	bool focused;
+};
+
+/*!
+ * Primary session state changes event.
+ *
+ * @ingroup xrt_iface
+ */
+struct xrt_compositor_event_overlay
+{
+	enum xrt_compositor_event_type type;
+	bool primary_focused;
+};
+
+/*!
+ * Compositor events union.
+ *
+ * @ingroup xrt_iface
+ */
+union xrt_compositor_event {
+	enum xrt_compositor_event_type type;
+	struct xrt_compositor_event_state_change state;
+	struct xrt_compositor_event_state_change overlay;
+};
+
+
+
+/*!
+ * Session prepare information, mostly overlay extension data.
+ *
+ * @ingroup xrt_iface
+ */
+struct xrt_session_prepare_info
+{
+	bool is_overlay;
+	uint64_t flags;
+	uint32_t z_order;
+};
+
 /*!
  * @interface xrt_compositor
  *
@@ -386,12 +441,14 @@ struct xrt_compositor
 	 *
 	 * This function is very much WIP.
 	 */
-	xrt_result_t (*poll_events)(struct xrt_compositor *xc, uint64_t *WIP);
+	xrt_result_t (*poll_events)(struct xrt_compositor *xc,
+	                            union xrt_compositor_event *out_xce);
 
 	/*!
 	 * This function is implicit in the OpenXR spec but made explicit here.
 	 */
-	xrt_result_t (*prepare_session)(struct xrt_compositor *xc);
+	xrt_result_t (*prepare_session)(struct xrt_compositor *xc,
+	                                struct xrt_session_prepare_info *xspi);
 
 	/*!
 	 * See xrBeginSession.
@@ -524,9 +581,10 @@ xrt_comp_create_swapchain(struct xrt_compositor *xc,
  * @public @memberof xrt_compositor
  */
 static inline xrt_result_t
-xrt_comp_poll_events(struct xrt_compositor *xc, uint64_t *WIP)
+xrt_comp_poll_events(struct xrt_compositor *xc,
+                     union xrt_compositor_event *out_xce)
 {
-	return xc->poll_events(xc, WIP);
+	return xc->poll_events(xc, out_xce);
 }
 
 /*!
@@ -537,9 +595,10 @@ xrt_comp_poll_events(struct xrt_compositor *xc, uint64_t *WIP)
  * @public @memberof xrt_compositor
  */
 static inline xrt_result_t
-xrt_comp_prepare_session(struct xrt_compositor *xc)
+xrt_comp_prepare_session(struct xrt_compositor *xc,
+                         struct xrt_session_prepare_info *xspi)
 {
-	return xc->prepare_session(xc);
+	return xc->prepare_session(xc, xspi);
 }
 
 /*!

src/xrt/include/xrt/xrt_instance.h

@@ -10,6 +10,8 @@
 #pragma once
 
 #include "xrt/xrt_compiler.h"
+#include "xrt/xrt_defines.h"
+
 
 #ifdef __cplusplus
 extern "C" {
@@ -20,11 +22,22 @@ struct xrt_prober;
 struct xrt_device;
 struct xrt_compositor_fd;
 
+
 /*!
  * @ingroup xrt_iface
  * @{
  */
 
+#define XRT_MAX_APPLICATION_NAME_SIZE 128
+
+/*!
+ * Information provided by the application at instance create time.
+ */
+struct xrt_instance_info
+{
+	char application_name[XRT_MAX_APPLICATION_NAME_SIZE];
+};
+
 /*!
  * @interface xrt_instance
  *
@@ -131,6 +144,7 @@ struct xrt_instance
 	/*!
 	 * @}
 	 */
+	struct xrt_instance_info instance_info;
 };
 
 /*!
@@ -220,7 +234,8 @@ xrt_instance_destroy(struct xrt_instance **xinst_ptr)
  * @relates xrt_instance
  */
 int
-xrt_instance_create(struct xrt_instance **out_xinst);
+xrt_instance_create(struct xrt_instance **out_xinst,
+                    struct xrt_instance_info *ii);
 
 /*!
  * @}

src/xrt/ipc/CMakeLists.txt

@@ -38,6 +38,7 @@ add_library(ipc_client STATIC
 	)
 target_include_directories(ipc_client INTERFACE
 	${CMAKE_CURRENT_SOURCE_DIR}
+	${CMAKE_CURRENT_BINARY_DIR}
 	)
 target_include_directories(ipc_client PRIVATE
 	${CMAKE_CURRENT_SOURCE_DIR}
@@ -65,6 +66,7 @@ add_library(ipc_server STATIC
 target_include_directories(ipc_server
 	INTERFACE
 	${CMAKE_CURRENT_SOURCE_DIR}
+	${CMAKE_CURRENT_BINARY_DIR}
 	)
 target_include_directories(ipc_server PRIVATE
 	${CMAKE_CURRENT_SOURCE_DIR}/../compositor

src/xrt/ipc/ipc_client_compositor.c

@@ -226,6 +226,31 @@ ipc_compositor_swapchain_create(struct xrt_compositor *xc,
 	return &ics->base.base;
 }
 
+static xrt_result_t
+ipc_compositor_prepare_session(struct xrt_compositor *xc,
+                               struct xrt_session_prepare_info *xspi)
+{
+	struct ipc_client_compositor *icc = ipc_client_compositor(xc);
+
+	IPC_SPEW(icc->ipc_c, "IPC: compositor create session");
+
+	IPC_CALL_CHK(ipc_call_session_create(icc->ipc_c, xspi));
+	return res;
+}
+
+static xrt_result_t
+ipc_compositor_poll_events(struct xrt_compositor *xc,
+                           union xrt_compositor_event *out_xce)
+{
+	struct ipc_client_compositor *icc = ipc_client_compositor(xc);
+
+	IPC_SPEW(icc->ipc_c, "IPC: polling for events");
+
+	IPC_CALL_CHK(ipc_call_compositor_poll_events(icc->ipc_c, out_xce));
+
+	return res;
+}
+
 static xrt_result_t
 ipc_compositor_begin_session(struct xrt_compositor *xc,
                              enum xrt_view_type view_type)
@@ -458,6 +483,7 @@ ipc_client_compositor_create(ipc_connection_t *ipc_c,
 	    U_TYPED_CALLOC(struct ipc_client_compositor);
 
 	c->base.base.create_swapchain = ipc_compositor_swapchain_create;
+	c->base.base.prepare_session = ipc_compositor_prepare_session;
 	c->base.base.begin_session = ipc_compositor_begin_session;
 	c->base.base.end_session = ipc_compositor_end_session;
 	c->base.base.wait_frame = ipc_compositor_wait_frame;
@@ -469,6 +495,7 @@ ipc_client_compositor_create(ipc_connection_t *ipc_c,
 	c->base.base.layer_quad = ipc_compositor_layer_quad;
 	c->base.base.layer_commit = ipc_compositor_layer_commit;
 	c->base.base.destroy = ipc_compositor_destroy;
+	c->base.base.poll_events = ipc_compositor_poll_events;
 	c->ipc_c = ipc_c;
 
 	// fetch our format list on client compositor construction

src/xrt/ipc/ipc_client_instance.c

@@ -183,7 +183,8 @@ ipc_client_instance_destroy(struct xrt_instance *xinst)
  * @public @memberof ipc_instance
  */
 int
-ipc_instance_create(struct xrt_instance **out_xinst)
+ipc_instance_create(struct xrt_instance **out_xinst,
+                    struct xrt_instance_info *i_info)
 {
 	struct ipc_client_instance *ii =
 	    U_TYPED_CALLOC(struct ipc_client_instance);
@@ -223,6 +224,17 @@ ipc_instance_create(struct xrt_instance **out_xinst)
 		return -1;
 	}
 
+	struct ipc_app_state desc = {0};
+	desc.info = *i_info;
+	desc.pid = getpid(); // Extra info.
+
+	r = ipc_call_system_set_client_info(&ii->ipc_c, &desc);
+	if (r != XRT_SUCCESS) {
+		IPC_ERROR(&ii->ipc_c, "Failed to set instance info");
+		free(ii);
+		return -1;
+	}
+
 	const int flags = MAP_SHARED;
 	const int access = PROT_READ | PROT_WRITE;
 	const size_t size = sizeof(struct ipc_shared_memory);

src/xrt/ipc/ipc_protocol.h

@@ -4,16 +4,20 @@
  * @file
  * @brief  Common protocol definition.
  * @author Pete Black <pblack@collabora.com>
+ * @author Jakob Bornecrantz <jakob@collabora.com>
  * @ingroup ipc
  */
 
 #pragma once
 
-#include "xrt/xrt_tracking.h"
-#include "xrt/xrt_device.h"
 #include "xrt/xrt_compiler.h"
 #include "xrt/xrt_compositor.h"
 #include "xrt/xrt_results.h"
+#include "xrt/xrt_defines.h"
+#include "xrt/xrt_instance.h"
+#include "xrt/xrt_compositor.h"
+#include "xrt/xrt_device.h"
+#include "xrt/xrt_tracking.h"
 
 
 #define IPC_MSG_SOCK_FILE "/tmp/monado_comp_ipc"
@@ -24,7 +28,9 @@
 #define IPC_MAX_FORMATS 32 // max formats our server-side compositor supports
 #define IPC_MAX_DEVICES 8  // max number of devices we will map via shared mem
 #define IPC_MAX_LAYERS 16
-#define IPC_MAX_SLOTS 3
+#define IPC_MAX_SLOTS 128
+#define IPC_MAX_CLIENTS 8
+#define IPC_EVENT_QUEUE_SIZE 32
 
 #define IPC_SHARED_MAX_DEVICES 8
 #define IPC_SHARED_MAX_INPUTS 1024
@@ -163,10 +169,24 @@ struct ipc_shared_memory
 	struct ipc_layer_slot slots[IPC_MAX_SLOTS];
 };
 
-/*
- *
- * Rest of protocol is generated.
+struct ipc_client_list
+{
+	int32_t ids[IPC_MAX_CLIENTS];
+};
+
+/*!
+ * State for a connected application.
  *
+ * @ingroup ipc
  */
-
-#include "ipc_protocol_generated.h"
+struct ipc_app_state
+{
+	bool primary_application;
+	bool session_active;
+	bool session_visible;
+	bool session_focused;
+	bool session_overlay;
+	uint32_t z_order;
+	pid_t pid;
+	struct xrt_instance_info info;
+};

src/xrt/ipc/ipc_server.h

@@ -64,7 +64,7 @@ extern "C" {
 
 #define IPC_SERVER_NUM_XDEVS 8
 #define IPC_MAX_CLIENT_SWAPCHAINS 32
-#define IPC_MAX_CLIENTS 8
+//#define IPC_MAX_CLIENTS 8
 
 struct xrt_instance;
 struct xrt_compositor;
@@ -86,6 +86,14 @@ struct ipc_swapchain_data
 	bool active;
 };
 
+
+struct ipc_queued_event
+{
+	bool pending;
+	uint64_t timestamp;
+	union xrt_compositor_event event;
+};
+
 /*!
  * Holds the state for a single client.
  *
@@ -117,7 +125,28 @@ struct ipc_client_state
 	//! Whether we are currently rendering @ref render_state
 	bool rendering_state;
 
-	bool active;
+	//! The frame timing state.
+	struct u_rt_helper urth;
+
+	struct ipc_app_state client_state;
+	struct ipc_queued_event queued_events[IPC_EVENT_QUEUE_SIZE];
+
+	int server_thread_index;
+};
+
+enum ipc_thread_state
+{
+	IPC_THREAD_READY,
+	IPC_THREAD_STARTING,
+	IPC_THREAD_RUNNING,
+	IPC_THREAD_STOPPING,
+};
+
+struct ipc_thread
+{
+	struct os_thread thread;
+	volatile enum ipc_thread_state state;
+	volatile struct ipc_client_state ics;
 };
 
 /*!
@@ -159,14 +188,13 @@ struct ipc_server
 	bool print_debug;
 	bool print_spew;
 
-	// Hack for now.
-	struct os_thread thread;
-	volatile bool thread_started;
-	volatile bool thread_stopping;
-	volatile struct ipc_client_state thread_state;
+	struct ipc_thread threads[IPC_MAX_CLIENTS];
 
+	volatile uint32_t current_slot_index;
 
-	struct u_rt_helper urth;
+	int active_client_index;
+	int last_active_client_index;
+	struct os_mutex global_state_lock;
 };
 
 /*!
@@ -177,6 +205,14 @@ struct ipc_server
 int
 ipc_server_main(int argc, char **argv);
 
+/*!
+ * Called by client threads to manage global state
+ *
+ * @ingroup ipc_server
+ */
+void
+update_server_state(struct ipc_server *vs);
+
 /*!
  * Thread function for the client side dispatching.
  *

src/xrt/ipc/proto.json

@@ -3,6 +3,45 @@
 		"out_fds": true
 	},
 
+	"system_get_client_info": {
+		"in": [
+			{"name": "id", "type": "uint32_t"}
+		],
+		"out": [
+			{"name": "desc", "type": "struct ipc_app_state"}
+		]
+	},
+
+	"system_set_client_info": {
+		"in": [
+			{"name": "desc", "type": "struct ipc_app_state"}
+		]
+	},
+
+	"system_get_clients": {
+		"out": [
+			{"name": "clients", "type": "struct ipc_client_list"}
+		]
+	},
+
+	"system_set_primary_client": {
+		"in": [
+			{"name": "id", "type": "uint32_t"}
+		]
+	},
+
+	"system_set_focused_client": {
+		"in": [
+			{"name": "id", "type": "uint32_t"}
+		]
+	},
+
+	"session_create": {
+		"in": [
+			{"name": "overlay_info", "type": "struct xrt_session_prepare_info"}
+		]
+	},
+
 	"session_begin": {},
 
 	"session_end": {},
@@ -51,6 +90,12 @@
 		]
 	},
 
+	"compositor_poll_events": {
+		"out": [
+			{"name": "event", "type": "union xrt_compositor_event"}
+		]
+	},
+
 	"swapchain_create": {
 		"in": [
 			{"name": "create", "type": "enum xrt_swapchain_create_flags"},

src/xrt/ipc/proto.py

@@ -246,6 +246,7 @@ def generate_client_c(file, p):
     f.write(header.format(brief='Generated IPC client code', suffix='_client'))
     f.write('''
 #include "ipc_client.h"
+#include "ipc_protocol_generated.h"
 
 
 // clang-format off
@@ -305,6 +306,7 @@ def generate_client_h(file, p):
 #pragma once
 
 #include "ipc_protocol.h"
+#include "ipc_protocol_generated.h"
 #include "ipc_client.h"
 
 
@@ -409,6 +411,7 @@ def generate_server_header(file, p):
 #pragma once
 
 #include "ipc_protocol.h"
+#include "ipc_protocol_generated.h"
 #include "ipc_server.h"
 
 

src/xrt/state_trackers/gui/gui_prober.c

@@ -38,7 +38,7 @@ gui_prober_init(struct gui_program *p)
 	int ret = 0;
 
 	// Initialize the prober.
-	ret = xrt_instance_create(&p->instance);
+	ret = xrt_instance_create(&p->instance, NULL);
 	if (ret != 0) {
 		return do_exit(p, ret);
 	}

src/xrt/state_trackers/oxr/oxr_event.c

@@ -157,6 +157,26 @@ oxr_event_push_XrEventDataSessionStateChanged(struct oxr_logger *log,
 	return XR_SUCCESS;
 }
 
+XrResult
+oxr_event_push_XrEventDataMainSessionVisibilityChangedEXTX(
+    struct oxr_logger *log, struct oxr_session *sess, bool visible)
+{
+	struct oxr_instance *inst = sess->sys->inst;
+	XrEventDataMainSessionVisibilityChangedEXTX *changed;
+	struct oxr_event *event = NULL;
+
+	ALLOC(log, inst, &event, &changed);
+	changed->type = XR_TYPE_EVENT_DATA_MAIN_SESSION_VISIBILITY_CHANGED_EXTX;
+	changed->flags = 0;
+	changed->visible = visible;
+	event->result = XR_SUCCESS;
+	lock(inst);
+	push(inst, event);
+	unlock(inst);
+
+	return XR_SUCCESS;
+}
+
 XrResult
 oxr_event_remove_session_events(struct oxr_logger *log,
                                 struct oxr_session *sess)
@@ -202,7 +222,7 @@ oxr_poll_event(struct oxr_logger *log,
 {
 	struct oxr_session *sess = inst->sessions;
 	while (sess) {
-		oxr_session_poll(sess);
+		oxr_session_poll(log, sess);
 		sess = sess->next;
 	}
 

src/xrt/state_trackers/oxr/oxr_extension_support.h

@@ -101,6 +101,16 @@
 #define OXR_EXTENSION_SUPPORT_MND_headless(_)
 #endif
 
+/*
+ * XR_EXTX_overlay
+ */
+#if defined(XR_EXTX_overlay)
+#define OXR_HAVE_EXTX_overlay
+#define OXR_EXTENSION_SUPPORT_EXTX_overlay(_) _(EXTX_overlay, EXTX_OVERLAY)
+#else
+#define OXR_EXTENSION_SUPPORT_EXTX_overlay(_)
+#endif
+
 // end of GENERATED per-extension defines - do not modify - used by scripts
 
 /*!
@@ -131,5 +141,6 @@
     OXR_EXTENSION_SUPPORT_KHR_opengl_es_enable(_) \
     OXR_EXTENSION_SUPPORT_KHR_vulkan_enable(_) \
     OXR_EXTENSION_SUPPORT_MNDX_egl_enable(_) \
-    OXR_EXTENSION_SUPPORT_MND_headless(_)
+    OXR_EXTENSION_SUPPORT_MND_headless(_) \
+    OXR_EXTENSION_SUPPORT_EXTX_overlay(_)
 // clang-format on

src/xrt/state_trackers/oxr/oxr_instance.c

@@ -7,11 +7,6 @@
  * @ingroup oxr_main
  */
 
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <math.h>
-
 #include "util/u_var.h"
 #include "util/u_time.h"
 #include "util/u_misc.h"
@@ -24,6 +19,13 @@
 #include "oxr_handle.h"
 #include "oxr_extension_support.h"
 
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <math.h>
+
 
 DEBUG_GET_ONCE_BOOL_OPTION(debug_views, "OXR_DEBUG_VIEWS", false)
 DEBUG_GET_ONCE_BOOL_OPTION(debug_spaces, "OXR_DEBUG_SPACES", false)
@@ -167,7 +169,15 @@ oxr_instance_create(struct oxr_logger *log,
 	cache_path(log, inst, "/interaction_profiles/mnd/ball_on_stick_controller", &inst->path_cache.mnd_ball_on_stick_controller);
 	// clang-format on
 
-	xinst_ret = xrt_instance_create(&inst->xinst);
+	// fill in our application info - @todo - replicate all createInfo
+	// fields?
+
+	struct xrt_instance_info i_info = {0};
+	snprintf(i_info.application_name,
+	         sizeof(inst->xinst->instance_info.application_name), "%s",
+	         createInfo->applicationInfo.applicationName);
+
+	xinst_ret = xrt_instance_create(&inst->xinst, &<

[Ryan Pavlik]

Can you document the instance info parameter here in the doxygen comment above? It's causing a build warning.

[Jakob Bornecrantz]

On it!