c/main: add hmd peek window

CMakeLists.txt

@@ -249,6 +249,7 @@ option_with_deps(XRT_FEATURE_SERVICE_SYSTEMD "Enable systemd socket activation o
 option_with_deps(XRT_FEATURE_SLAM "Enable SLAM tracking support" DEPENDS XRT_HAVE_OPENCV "XRT_HAVE_BASALT_SLAM OR XRT_HAVE_KIMERA_SLAM")
 option_with_deps(XRT_FEATURE_STEAMVR_PLUGIN "Build SteamVR plugin" DEPENDS "NOT ANDROID")
 option_with_deps(XRT_FEATURE_TRACING "Enable debug tracing on supported platforms" DEFAULT OFF DEPENDS XRT_HAVE_PERCETTO)
+option_with_deps(XRT_FEATURE_WINDOW_PEEK "Enable a window that displays the content of the HMD on screen" DEPENDS XRT_HAVE_SDL2)
 
 if (XRT_FEATURE_SERVICE)
 	# Disable the client debug gui by default for out-of-proc -

src/xrt/compositor/CMakeLists.txt

@@ -212,6 +212,10 @@ if(XRT_FEATURE_COMPOSITOR_MAIN)
 			)
 		target_link_libraries(comp_main PRIVATE ${X11_X11_LIB})
 	endif()
+	if(XRT_FEATURE_WINDOW_PEEK)
+		target_sources(comp_main PRIVATE main/comp_window_peek.c)
+		target_link_libraries(comp_main PRIVATE ${SDL2_LIBRARIES})
+	endif()
 	if(WIN32)
 		target_sources(comp_main PRIVATE main/comp_window_mswin.c)
 	endif()

src/xrt/compositor/main/comp_compositor.c

@@ -60,6 +60,10 @@
 #include "util/comp_vulkan.h"
 #include "main/comp_compositor.h"
 
+#ifdef XRT_FEATURE_WINDOW_PEEK
+#include "main/comp_window_peek.h"
+#endif
+
 #include "multi/comp_multi_interface.h"
 
 #include <math.h>
@@ -344,7 +348,7 @@ compositor_layer_commit(struct xrt_compositor *xc, int64_t frame_id, xrt_graphic
 	 * We have a fast path for single projection layer that goes directly
 	 * to the distortion shader, so no need to use the layer renderer.
 	 */
-	bool fast_path = can_do_one_projection_layer_fast_path(c) && !c->mirroring_to_debug_gui;
+	bool fast_path = can_do_one_projection_layer_fast_path(c) && !c->mirroring_to_debug_gui && !c->peek;
 	c->base.slot.one_projection_layer_fast_path = fast_path;
 
 
@@ -420,6 +424,10 @@ compositor_destroy(struct xrt_compositor *xc)
 
 	comp_renderer_destroy(&c->r);
 
+#ifdef XRT_FEATURE_WINDOW_PEEK
+	comp_window_peek_destroy(&c->peek);
+#endif
+
 	render_resources_close(&c->nr);
 
 	// As long as vk_bundle is valid it's safe to call this function.
@@ -1124,11 +1132,16 @@ compositor_init_renderer(struct comp_compositor *c)
 	}
 
 	c->r = comp_renderer_create(c);
+
+#ifdef XRT_FEATURE_WINDOW_PEEK
+	c->peek = comp_window_peek_create(c);
+#else
+	c->peek = NULL;
+#endif
+
 	return c->r != NULL;
 }
 
-
-
 xrt_result_t
 xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compositor **out_xsysc)
 {

src/xrt/compositor/main/comp_compositor.h

@@ -29,6 +29,7 @@
 #include "main/comp_settings.h"
 #include "main/comp_renderer.h"
 
+struct comp_window_peek;
 
 #ifdef __cplusplus
 extern "C" {
@@ -105,6 +106,9 @@ struct comp_compositor
 	//! Are we mirroring any of the views to the debug gui? If so, turn off the fast path.
 	bool mirroring_to_debug_gui;
 
+	//! On screen window to display the content of the HMD.
+	struct comp_window_peek *peek;
+
 	/*!
 	 * @brief Data exclusive to the begin_frame/end_frame for computing an
 	 * estimate of the app's needs.

src/xrt/compositor/main/comp_renderer.c

@@ -31,6 +31,9 @@
 
 #include "main/comp_layer_renderer.h"
 
+#ifdef XRT_FEATURE_WINDOW_PEEK
+#include "main/comp_window_peek.h"
+#endif
 
 #include "vk/vk_helpers.h"
 #include "vk/vk_image_readback_to_xf_pool.h"
@@ -518,9 +521,13 @@ renderer_ensure_images_and_renderings(struct comp_renderer *r, bool force_recrea
 
 	VkImageUsageFlags image_usage = 0;
 	if (r->settings->use_compute) {
-		image_usage = VK_IMAGE_USAGE_STORAGE_BIT;
+		image_usage |= VK_IMAGE_USAGE_STORAGE_BIT;
 	} else {
-		image_usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+		image_usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+	}
+
+	if (c->peek) {
+		image_usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
 	}
 
 	comp_target_create_images(           //
@@ -1538,6 +1545,26 @@ comp_renderer_draw(struct comp_renderer *r)
 		dispatch_graphics(r, &rr);
 	}
 
+#ifdef XRT_FEATURE_WINDOW_PEEK
+	if (c->peek) {
+		switch (c->peek->eye) {
+		case COMP_WINDOW_PEEK_EYE_LEFT:
+			comp_window_peek_blit(c->peek, r->lr->framebuffers[0].image, r->lr->extent.width,
+			                      r->lr->extent.height);
+			break;
+		case COMP_WINDOW_PEEK_EYE_RIGHT:
+			comp_window_peek_blit(c->peek, r->lr->framebuffers[1].image, r->lr->extent.width,
+			                      r->lr->extent.height);
+			break;
+		case COMP_WINDOW_PEEK_EYE_BOTH:
+			/* TODO: display the undistorted image */
+			comp_window_peek_blit(c->peek, c->target->images[r->acquired_buffer].handle, c->target->width,
+			                      c->target->height);
+			break;
+		}
+	}
+#endif
+
 	renderer_present_swapchain_image(r, c->frame.rendering.desired_present_time_ns,
 	                                 c->frame.rendering.present_slop_ns);
 

src/xrt/compositor/main/comp_window_peek.c

@@ -0,0 +1,376 @@
+// Copyright 2022, Simon Zeni <simon@bl4ckb0ne.ca>
+// SPDX-License-Identifier: BSL-1.0
+/*!
+ * @file
+ * @brief Displays the content of one or both eye onto a desktop window
+ * @author Simon Zeni <simon@bl4ckb0ne.ca>
+ * @ingroup comp_main
+ */
+
+#include "main/comp_compositor.h"
+#include "main/comp_renderer.h"
+#include "main/comp_target_swapchain.h"
+#include "main/comp_window_peek.h"
+
+#include "util/u_debug.h"
+
+#include <SDL2/SDL_vulkan.h>
+
+DEBUG_GET_ONCE_OPTION(window_peek, "XRT_WINDOW_PEEK", NULL)
+
+static inline struct vk_bundle *
+get_vk(struct comp_window_peek *w)
+{
+	return &w->c->base.vk;
+}
+
+static void *
+window_peek_run_thread(void *ptr)
+{
+	struct comp_window_peek *w = ptr;
+
+	w->running = true;
+	w->hidden = false;
+	while (w->running) {
+		SDL_Event event;
+		while (SDL_PollEvent(&event)) {
+			switch (event.type) {
+			case SDL_QUIT: w->running = false; break;
+			case SDL_WINDOWEVENT:
+				switch (event.window.event) {
+				case SDL_WINDOWEVENT_HIDDEN: w->hidden = true; break;
+				case SDL_WINDOWEVENT_SHOWN: w->hidden = false; break;
+				case SDL_WINDOWEVENT_SIZE_CHANGED:
+					w->width = event.window.data1;
+					w->height = event.window.data2;
+					break;
+#if SDL_VERSION_ATLEAST(2, 0, 18)
+				case SDL_WINDOWEVENT_DISPLAY_CHANGED:
+#endif
+				case SDL_WINDOWEVENT_MOVED:
+					SDL_GetWindowSize(w->window, (int *)&w->width, (int *)&w->height);
+					break;
+				default: break;
+				}
+				break;
+			case SDL_KEYDOWN:
+				switch (event.key.keysym.sym) {
+				case SDLK_ESCAPE: w->running = false; break;
+				default: break;
+				}
+				break;
+			default: break;
+			}
+
+			if (event.type == SDL_QUIT) {
+				w->running = false;
+			}
+		}
+	}
+
+	return NULL;
+}
+
+struct comp_window_peek *
+comp_window_peek_create(struct comp_compositor *c)
+{
+	const char *compute = getenv("XRT_COMPOSITOR_COMPUTE");
+	if (compute) {
+		COMP_WARN(c, "Peek window cannot be enabled on compute compositor");
+		return NULL;
+	}
+
+	const char *option = debug_get_option_window_peek();
+	if (option == NULL) {
+		return NULL;
+	}
+
+	struct xrt_device *xdev = c->xdev;
+	enum comp_window_peek_eye eye = -1;
+
+	int32_t width, height;
+	if (strcmp(option, "both") == 0 || strcmp(option, "BOTH") == 0 || strcmp(option, "") == 0) {
+		eye = COMP_WINDOW_PEEK_EYE_BOTH;
+		width = xdev->hmd->screens[0].w_pixels;
+		height = xdev->hmd->screens[0].h_pixels;
+	} else if (strcmp(option, "left") == 0 || strcmp(option, "LEFT") == 0) {
+		eye = COMP_WINDOW_PEEK_EYE_LEFT;
+		width = xdev->hmd->views[0].display.w_pixels;
+		height = xdev->hmd->views[0].display.h_pixels;
+	} else if (strcmp(option, "right") == 0 || strcmp(option, "RIGHT") == 0) {
+		eye = COMP_WINDOW_PEEK_EYE_RIGHT;
+		width = xdev->hmd->views[1].display.w_pixels;
+		height = xdev->hmd->views[1].display.h_pixels;
+	} else {
+		COMP_ERROR(c, "XRT_window_peek invalid option '%s'", option);
+		COMP_ERROR(c, "must be one of 'both', 'left' or 'right'");
+		return NULL;
+	}
+
+	COMP_DEBUG(c, "Creating peek window from %s eye(s)", option);
+
+	struct comp_window_peek *w = U_TYPED_CALLOC(struct comp_window_peek);
+	w->c = c;
+	w->eye = eye;
+
+	if (SDL_Init(SDL_INIT_VIDEO) < 0) {
+		COMP_ERROR(c, "Failed to init SDL2");
+		return NULL;
+	}
+
+	int x = SDL_WINDOWPOS_UNDEFINED;
+	int y = SDL_WINDOWPOS_UNDEFINED;
+
+	int flags = SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE | SDL_WINDOW_VULKAN;
+
+	w->window = SDL_CreateWindow(xdev->str, x, y, width, height, flags);
+	if (w->window == NULL) {
+		COMP_ERROR(c, "Failed to create SDL window: %s", SDL_GetError());
+		free(w);
+		return NULL;
+	}
+
+	w->width = width;
+	w->height = height;
+
+	comp_target_swapchain_init_and_set_fnptrs(&w->base, COMP_TARGET_FORCE_FAKE_DISPLAY_TIMING);
+
+	struct vk_bundle *vk = get_vk(w);
+
+	w->base.base.name = "peek";
+	w->base.base.c = c;
+	w->base.display = VK_NULL_HANDLE;
+
+	if (!SDL_Vulkan_CreateSurface(w->window, vk->instance, &w->base.surface.handle)) {
+		COMP_ERROR(c, "Failed to create SDL surface: %s", SDL_GetError());
+		SDL_DestroyWindow(w->window);
+		free(w);
+		return NULL;
+	}
+
+	/* TODO: present mode fallback to FIFO if MAILBOX is not available */
+	comp_target_create_images(&w->base.base, w->width, w->height, w->c->settings.color_format,
+	                          w->c->settings.color_space, VK_IMAGE_USAGE_TRANSFER_DST_BIT,
+	                          VK_PRESENT_MODE_MAILBOX_KHR);
+
+	VkSemaphoreCreateInfo sem_info = {
+	    .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
+	    .pNext = NULL,
+	    .flags = 0,
+	};
+
+	VkResult ret = vk->vkCreateSemaphore(vk->device, &sem_info, NULL, &w->acquire);
+	if (ret != VK_SUCCESS) {
+		COMP_ERROR(c, "vkCreateSemaphore: %s", vk_result_string(ret));
+	}
+
+	ret = vk->vkCreateSemaphore(vk->device, &sem_info, NULL, &w->submit);
+	if (ret != VK_SUCCESS) {
+		COMP_ERROR(c, "vkCreateSemaphore: %s", vk_result_string(ret));
+	}
+
+	os_thread_helper_init(&w->oth);
+	os_thread_helper_start(&w->oth, window_peek_run_thread, w);
+
+	return w;
+}
+
+void
+comp_window_peek_destroy(struct comp_window_peek **w_ptr)
+{
+	struct comp_window_peek *w = *w_ptr;
+	if (w == NULL) {
+		return;
+	}
+
+	w->running = false;
+
+	SDL_DestroyWindow(w->window);
+
+	comp_target_swapchain_cleanup(&w->base);
+	os_thread_helper_destroy(&w->oth);
+
+	free(w);
+
+	*w_ptr = NULL;
+}
+
+void
+comp_window_peek_blit(struct comp_window_peek *w, VkImage src, int32_t width, int32_t height)
+{
+	if (w->hidden) {
+		return;
+	}
+
+	if (w->width != w->base.base.width || w->height != w->base.base.height) {
+		COMP_DEBUG(w->c, "Resizing swapchain");
+		comp_target_create_images(&w->base.base, w->width, w->height, w->c->settings.color_format,
+		                          w->c->settings.color_space, VK_IMAGE_USAGE_TRANSFER_DST_BIT,
+		                          VK_PRESENT_MODE_MAILBOX_KHR);
+	}
+
+	while (!comp_target_check_ready(&w->base.base))
+		;
+
+	uint32_t current;
+	VkResult ret = comp_target_acquire(&w->base.base, w->acquire, &current);
+	if (ret != VK_SUCCESS) {
+		COMP_ERROR(w->c, "comp_target_acquire: %s", vk_result_string(ret));
+	}
+
+	VkImage dst = w->base.base.images[current].handle;
+
+	struct vk_bundle *vk = get_vk(w);
+
+	VkCommandBuffer cmd;
+	vk_init_cmd_buffer(vk, &cmd);
+
+	// For submitting commands.
+	os_mutex_lock(&vk->cmd_pool_mutex);
+
+	VkImageSubresourceRange range = {
+	    .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+	    .baseMipLevel = 0,
+	    .levelCount = 1,
+	    .baseArrayLayer = 0,
+	    .layerCount = 1,
+	};
+
+	// Barrier to make source a source
+	vk_cmd_image_barrier_locked(                       //
+	    vk,                                            // vk_bundle
+	    cmd,                                           // cmdbuffer
+	    src,                                           // image
+	    VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,          // srcAccessMask
+	    VK_ACCESS_TRANSFER_READ_BIT,                   // dstAccessMask
+	    VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,      // oldImageLayout
+	    VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,          // newImageLayout
+	    VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, // srcStageMask
+	    VK_PIPELINE_STAGE_TRANSFER_BIT,                // dstStageMask
+	    range);                                        // subresourceRange
+
+	// Barrier to make destination a destination
+	vk_cmd_image_barrier_locked(              //
+	    vk,                                   // vk_bundle
+	    cmd,                                  // cmdbuffer
+	    dst,                                  // image
+	    0,                                    // srcAccessMask
+	    VK_ACCESS_TRANSFER_WRITE_BIT,         // dstAccessMask
+	    VK_IMAGE_LAYOUT_UNDEFINED,            // oldImageLayout
+	    VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, // newImageLayout
+	    VK_PIPELINE_STAGE_TRANSFER_BIT,       // srcStageMask
+	    VK_PIPELINE_STAGE_TRANSFER_BIT,       // dstStageMask
+	    range);                               // subresourceRange
+
+	VkImageBlit blit = {.srcSubresource =
+	                        {
+	                            .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+	                            .layerCount = 1,
+	                        },
+	                    .dstSubresource = {
+	                        .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+	                        .layerCount = 1,
+	                    }};
+
+	blit.srcOffsets[1].x = width;
+	blit.srcOffsets[1].y = height;
+	blit.srcOffsets[1].z = 1;
+
+	blit.dstOffsets[1].x = w->width;
+	blit.dstOffsets[1].y = w->height;
+	blit.dstOffsets[1].z = 1;
+
+	vk->vkCmdBlitImage(                       //
+	    cmd,                                  // commandBuffer
+	    src,                                  // srcImage
+	    VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, // srcImageLayout
+	    dst,                                  // dstImage
+	    VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, // dstImageLayout
+	    1,                                    // regionCount
+	    &blit,                                // pRegions
+	    VK_FILTER_LINEAR                      // filter
+	);
+
+	// Reset destination
+	vk_cmd_image_barrier_locked(              //
+	    vk,                                   // vk_bundle
+	    cmd,                                  // cmdbuffer
+	    dst,                                  // image
+	    VK_ACCESS_TRANSFER_WRITE_BIT,         // srcAccessMask
+	    0,                                    // dstAccessMask
+	    VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, // oldImageLayout
+	    VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,      // newImageLayout
+	    VK_PIPELINE_STAGE_TRANSFER_BIT,       // srcStageMask
+	    VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, // dstStageMask
+	    range);                               // subresourceRange
+
+	// Reset src
+	vk_cmd_image_barrier_locked(                  //
+	    vk,                                       // vk_bundle
+	    cmd,                                      // cmdbuffer
+	    src,                                      // image
+	    VK_ACCESS_TRANSFER_READ_BIT,              // srcAccessMask
+	    0,                                        // dstAccessMask
+	    VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,     // oldImageLayout
+	    VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, // newImageLayout
+	    VK_PIPELINE_STAGE_TRANSFER_BIT,           // srcStageMask
+	    VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,        // dstStageMask
+	    range);                                   // subresourceRange
+
+	ret = vk->vkEndCommandBuffer(cmd);
+
+	if (ret != VK_SUCCESS) {
+		VK_ERROR(vk, "Error: Could not end command buffer.\n");
+		vk->vkFreeCommandBuffers(vk->device, vk->cmd_pool, 1, &cmd);
+		os_mutex_unlock(&vk->cmd_pool_mutex);
+		return;
+	}
+
+	os_mutex_unlock(&vk->cmd_pool_mutex);
+
+	VkPipelineStageFlags submit_flags = VK_PIPELINE_STAGE_TRANSFER_BIT;
+
+	// Waits for command to finish.
+	VkSubmitInfo submit = {
+	    .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
+	    .pNext = NULL,
+	    .waitSemaphoreCount = 1,
+	    .pWaitSemaphores = &w->acquire,
+	    .pWaitDstStageMask = &submit_flags,
+	    .commandBufferCount = 1,
+	    .pCommandBuffers = &cmd,
+	    .signalSemaphoreCount = 1,
+	    .pSignalSemaphores = &w->submit,
+	};
+
+	os_mutex_lock(&vk->queue_mutex);
+	os_mutex_lock(&vk->cmd_pool_mutex);
+	ret = vk->vkQueueSubmit(vk->queue, 1, &submit, VK_NULL_HANDLE);
+	os_mutex_unlock(&vk->cmd_pool_mutex);
+	os_mutex_unlock(&vk->queue_mutex);
+
+	if (ret != VK_SUCCESS) {
+		VK_ERROR(vk, "Error: Could not submit to queue.\n");
+		return;
+	}
+
+	VkPresentInfoKHR present = {
+	    .sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR,
+	    .pNext = NULL,
+	    .waitSemaphoreCount = 1,
+	    .pWaitSemaphores = &w->submit,
+	    .swapchainCount = 1,
+	    .pSwapchains = &w->base.swapchain.handle,
+	    .pImageIndices = &current,
+	    .pResults = NULL,
+	};
+
+	os_mutex_lock(&vk->queue_mutex);
+	ret = vk->vkQueuePresentKHR(vk->queue, &present);
+	os_mutex_unlock(&vk->queue_mutex);
+
+	if (ret != VK_SUCCESS) {
+		VK_ERROR(vk, "Error: could not present to queue.\n");
+		return;
+	}
+}

src/xrt/compositor/main/comp_window_peek.h

@@ -0,0 +1,69 @@
+// Copyright 2022, Simon Zeni <simon@bl4ckb0ne.ca>
+// SPDX-License-Identifier: BSL-1.0
+/*!
+ * @file
+ * @brief Displays the content of one or both eye onto a desktop window
+ * @author Simon Zeni <simon@bl4ckb0ne.ca>
+ * @ingroup comp_main
+ */
+
+#pragma once
+
+#include "xrt/xrt_config_have.h"
+
+#ifndef XRT_FEATURE_WINDOW_PEEK
+#error "XRT_FEATURE_WINDOW_PEEK not enabled"
+#endif
+
+#include "os/os_threading.h"
+
+#ifdef XRT_HAVE_SDL2
+#include <SDL2/SDL.h>
+#else
+#error "comp_window_peek.h requires SDL2"
+#endif
+
+struct comp_compositor;
+struct comp_renderer;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum comp_window_peek_eye
+{
+	COMP_WINDOW_PEEK_EYE_LEFT = 0,
+	COMP_WINDOW_PEEK_EYE_RIGHT = 1,
+	COMP_WINDOW_PEEK_EYE_BOTH = 2,
+};
+
+struct comp_window_peek
+{
+	struct comp_target_swapchain base;
+	struct comp_compositor *c;
+
+	enum comp_window_peek_eye eye;
+	SDL_Window *window;
+	uint32_t width, height;
+	bool running;
+	bool hidden;
+
+	VkSurfaceKHR surface;
+	VkSemaphore acquire;
+	VkSemaphore submit;
+
+	struct os_thread_helper oth;
+};
+
+struct comp_window_peek *
+comp_window_peek_create(struct comp_compositor *c);
+
+void
+comp_window_peek_destroy(struct comp_window_peek **w_ptr);
+
+void
+comp_window_peek_blit(struct comp_window_peek *w, VkImage src, int32_t width, int32_t height);
+
+#ifdef __cplusplus
+}
+#endif

src/xrt/include/xrt/xrt_config_build.h.cmake_in

@@ -27,3 +27,4 @@
 #cmakedefine XRT_FEATURE_SLAM
 #cmakedefine XRT_FEATURE_TRACING
 #cmakedefine XRT_FEATURE_CLIENT_DEBUG_GUI
+#cmakedefine XRT_FEATURE_WINDOW_PEEK