From 778900cd3ce264307d43c82617b15531d468af55 Mon Sep 17 00:00:00 2001 From: Simon Zeni Date: Thu, 7 Jul 2022 16:10:56 -0400 Subject: [PATCH] c/main: add hmd peek window --- CMakeLists.txt | 1 + src/xrt/compositor/CMakeLists.txt | 4 + src/xrt/compositor/main/comp_compositor.c | 19 +- src/xrt/compositor/main/comp_compositor.h | 4 + src/xrt/compositor/main/comp_renderer.c | 31 +- src/xrt/compositor/main/comp_window_peek.c | 376 ++++++++++++++++++ src/xrt/compositor/main/comp_window_peek.h | 69 ++++ .../include/xrt/xrt_config_build.h.cmake_in | 1 + 8 files changed, 500 insertions(+), 5 deletions(-) create mode 100644 src/xrt/compositor/main/comp_window_peek.c create mode 100644 src/xrt/compositor/main/comp_window_peek.h diff --git a/CMakeLists.txt b/CMakeLists.txt index c490d3f19..e286d4852 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -249,6 +249,7 @@ option_with_deps(XRT_FEATURE_SERVICE_SYSTEMD "Enable systemd socket activation o option_with_deps(XRT_FEATURE_SLAM "Enable SLAM tracking support" DEPENDS XRT_HAVE_OPENCV "XRT_HAVE_BASALT_SLAM OR XRT_HAVE_KIMERA_SLAM") option_with_deps(XRT_FEATURE_STEAMVR_PLUGIN "Build SteamVR plugin" DEPENDS "NOT ANDROID") option_with_deps(XRT_FEATURE_TRACING "Enable debug tracing on supported platforms" DEFAULT OFF DEPENDS XRT_HAVE_PERCETTO) +option_with_deps(XRT_FEATURE_WINDOW_PEEK "Enable a window that displays the content of the HMD on screen" DEPENDS XRT_HAVE_SDL2) if (XRT_FEATURE_SERVICE) # Disable the client debug gui by default for out-of-proc - diff --git a/src/xrt/compositor/CMakeLists.txt b/src/xrt/compositor/CMakeLists.txt index a2242105f..d40b5edda 100644 --- a/src/xrt/compositor/CMakeLists.txt +++ b/src/xrt/compositor/CMakeLists.txt @@ -212,6 +212,10 @@ if(XRT_FEATURE_COMPOSITOR_MAIN) ) target_link_libraries(comp_main PRIVATE ${X11_X11_LIB}) endif() + if(XRT_FEATURE_WINDOW_PEEK) + target_sources(comp_main PRIVATE main/comp_window_peek.c) + target_link_libraries(comp_main PRIVATE ${SDL2_LIBRARIES}) + endif() if(WIN32) target_sources(comp_main PRIVATE main/comp_window_mswin.c) endif() diff --git a/src/xrt/compositor/main/comp_compositor.c b/src/xrt/compositor/main/comp_compositor.c index c984d9b05..cdae59d8a 100644 --- a/src/xrt/compositor/main/comp_compositor.c +++ b/src/xrt/compositor/main/comp_compositor.c @@ -60,6 +60,10 @@ #include "util/comp_vulkan.h" #include "main/comp_compositor.h" +#ifdef XRT_FEATURE_WINDOW_PEEK +#include "main/comp_window_peek.h" +#endif + #include "multi/comp_multi_interface.h" #include @@ -344,7 +348,7 @@ compositor_layer_commit(struct xrt_compositor *xc, int64_t frame_id, xrt_graphic * We have a fast path for single projection layer that goes directly * to the distortion shader, so no need to use the layer renderer. */ - bool fast_path = can_do_one_projection_layer_fast_path(c) && !c->mirroring_to_debug_gui; + bool fast_path = can_do_one_projection_layer_fast_path(c) && !c->mirroring_to_debug_gui && !c->peek; c->base.slot.one_projection_layer_fast_path = fast_path; @@ -420,6 +424,10 @@ compositor_destroy(struct xrt_compositor *xc) comp_renderer_destroy(&c->r); +#ifdef XRT_FEATURE_WINDOW_PEEK + comp_window_peek_destroy(&c->peek); +#endif + render_resources_close(&c->nr); // As long as vk_bundle is valid it's safe to call this function. @@ -1124,11 +1132,16 @@ compositor_init_renderer(struct comp_compositor *c) } c->r = comp_renderer_create(c); + +#ifdef XRT_FEATURE_WINDOW_PEEK + c->peek = comp_window_peek_create(c); +#else + c->peek = NULL; +#endif + return c->r != NULL; } - - xrt_result_t xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compositor **out_xsysc) { diff --git a/src/xrt/compositor/main/comp_compositor.h b/src/xrt/compositor/main/comp_compositor.h index 792eef72c..f3f567adb 100644 --- a/src/xrt/compositor/main/comp_compositor.h +++ b/src/xrt/compositor/main/comp_compositor.h @@ -29,6 +29,7 @@ #include "main/comp_settings.h" #include "main/comp_renderer.h" +struct comp_window_peek; #ifdef __cplusplus extern "C" { @@ -105,6 +106,9 @@ struct comp_compositor //! Are we mirroring any of the views to the debug gui? If so, turn off the fast path. bool mirroring_to_debug_gui; + //! On screen window to display the content of the HMD. + struct comp_window_peek *peek; + /*! * @brief Data exclusive to the begin_frame/end_frame for computing an * estimate of the app's needs. diff --git a/src/xrt/compositor/main/comp_renderer.c b/src/xrt/compositor/main/comp_renderer.c index cfe2bf2ef..492aff304 100644 --- a/src/xrt/compositor/main/comp_renderer.c +++ b/src/xrt/compositor/main/comp_renderer.c @@ -31,6 +31,9 @@ #include "main/comp_layer_renderer.h" +#ifdef XRT_FEATURE_WINDOW_PEEK +#include "main/comp_window_peek.h" +#endif #include "vk/vk_helpers.h" #include "vk/vk_image_readback_to_xf_pool.h" @@ -518,9 +521,13 @@ renderer_ensure_images_and_renderings(struct comp_renderer *r, bool force_recrea VkImageUsageFlags image_usage = 0; if (r->settings->use_compute) { - image_usage = VK_IMAGE_USAGE_STORAGE_BIT; + image_usage |= VK_IMAGE_USAGE_STORAGE_BIT; } else { - image_usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + image_usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + } + + if (c->peek) { + image_usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; } comp_target_create_images( // @@ -1538,6 +1545,26 @@ comp_renderer_draw(struct comp_renderer *r) dispatch_graphics(r, &rr); } +#ifdef XRT_FEATURE_WINDOW_PEEK + if (c->peek) { + switch (c->peek->eye) { + case COMP_WINDOW_PEEK_EYE_LEFT: + comp_window_peek_blit(c->peek, r->lr->framebuffers[0].image, r->lr->extent.width, + r->lr->extent.height); + break; + case COMP_WINDOW_PEEK_EYE_RIGHT: + comp_window_peek_blit(c->peek, r->lr->framebuffers[1].image, r->lr->extent.width, + r->lr->extent.height); + break; + case COMP_WINDOW_PEEK_EYE_BOTH: + /* TODO: display the undistorted image */ + comp_window_peek_blit(c->peek, c->target->images[r->acquired_buffer].handle, c->target->width, + c->target->height); + break; + } + } +#endif + renderer_present_swapchain_image(r, c->frame.rendering.desired_present_time_ns, c->frame.rendering.present_slop_ns); diff --git a/src/xrt/compositor/main/comp_window_peek.c b/src/xrt/compositor/main/comp_window_peek.c new file mode 100644 index 000000000..53cc700f9 --- /dev/null +++ b/src/xrt/compositor/main/comp_window_peek.c @@ -0,0 +1,376 @@ +// Copyright 2022, Simon Zeni +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Displays the content of one or both eye onto a desktop window + * @author Simon Zeni + * @ingroup comp_main + */ + +#include "main/comp_compositor.h" +#include "main/comp_renderer.h" +#include "main/comp_target_swapchain.h" +#include "main/comp_window_peek.h" + +#include "util/u_debug.h" + +#include + +DEBUG_GET_ONCE_OPTION(window_peek, "XRT_WINDOW_PEEK", NULL) + +static inline struct vk_bundle * +get_vk(struct comp_window_peek *w) +{ + return &w->c->base.vk; +} + +static void * +window_peek_run_thread(void *ptr) +{ + struct comp_window_peek *w = ptr; + + w->running = true; + w->hidden = false; + while (w->running) { + SDL_Event event; + while (SDL_PollEvent(&event)) { + switch (event.type) { + case SDL_QUIT: w->running = false; break; + case SDL_WINDOWEVENT: + switch (event.window.event) { + case SDL_WINDOWEVENT_HIDDEN: w->hidden = true; break; + case SDL_WINDOWEVENT_SHOWN: w->hidden = false; break; + case SDL_WINDOWEVENT_SIZE_CHANGED: + w->width = event.window.data1; + w->height = event.window.data2; + break; +#if SDL_VERSION_ATLEAST(2, 0, 18) + case SDL_WINDOWEVENT_DISPLAY_CHANGED: +#endif + case SDL_WINDOWEVENT_MOVED: + SDL_GetWindowSize(w->window, (int *)&w->width, (int *)&w->height); + break; + default: break; + } + break; + case SDL_KEYDOWN: + switch (event.key.keysym.sym) { + case SDLK_ESCAPE: w->running = false; break; + default: break; + } + break; + default: break; + } + + if (event.type == SDL_QUIT) { + w->running = false; + } + } + } + + return NULL; +} + +struct comp_window_peek * +comp_window_peek_create(struct comp_compositor *c) +{ + const char *compute = getenv("XRT_COMPOSITOR_COMPUTE"); + if (compute) { + COMP_WARN(c, "Peek window cannot be enabled on compute compositor"); + return NULL; + } + + const char *option = debug_get_option_window_peek(); + if (option == NULL) { + return NULL; + } + + struct xrt_device *xdev = c->xdev; + enum comp_window_peek_eye eye = -1; + + int32_t width, height; + if (strcmp(option, "both") == 0 || strcmp(option, "BOTH") == 0 || strcmp(option, "") == 0) { + eye = COMP_WINDOW_PEEK_EYE_BOTH; + width = xdev->hmd->screens[0].w_pixels; + height = xdev->hmd->screens[0].h_pixels; + } else if (strcmp(option, "left") == 0 || strcmp(option, "LEFT") == 0) { + eye = COMP_WINDOW_PEEK_EYE_LEFT; + width = xdev->hmd->views[0].display.w_pixels; + height = xdev->hmd->views[0].display.h_pixels; + } else if (strcmp(option, "right") == 0 || strcmp(option, "RIGHT") == 0) { + eye = COMP_WINDOW_PEEK_EYE_RIGHT; + width = xdev->hmd->views[1].display.w_pixels; + height = xdev->hmd->views[1].display.h_pixels; + } else { + COMP_ERROR(c, "XRT_window_peek invalid option '%s'", option); + COMP_ERROR(c, "must be one of 'both', 'left' or 'right'"); + return NULL; + } + + COMP_DEBUG(c, "Creating peek window from %s eye(s)", option); + + struct comp_window_peek *w = U_TYPED_CALLOC(struct comp_window_peek); + w->c = c; + w->eye = eye; + + if (SDL_Init(SDL_INIT_VIDEO) < 0) { + COMP_ERROR(c, "Failed to init SDL2"); + return NULL; + } + + int x = SDL_WINDOWPOS_UNDEFINED; + int y = SDL_WINDOWPOS_UNDEFINED; + + int flags = SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE | SDL_WINDOW_VULKAN; + + w->window = SDL_CreateWindow(xdev->str, x, y, width, height, flags); + if (w->window == NULL) { + COMP_ERROR(c, "Failed to create SDL window: %s", SDL_GetError()); + free(w); + return NULL; + } + + w->width = width; + w->height = height; + + comp_target_swapchain_init_and_set_fnptrs(&w->base, COMP_TARGET_FORCE_FAKE_DISPLAY_TIMING); + + struct vk_bundle *vk = get_vk(w); + + w->base.base.name = "peek"; + w->base.base.c = c; + w->base.display = VK_NULL_HANDLE; + + if (!SDL_Vulkan_CreateSurface(w->window, vk->instance, &w->base.surface.handle)) { + COMP_ERROR(c, "Failed to create SDL surface: %s", SDL_GetError()); + SDL_DestroyWindow(w->window); + free(w); + return NULL; + } + + /* TODO: present mode fallback to FIFO if MAILBOX is not available */ + comp_target_create_images(&w->base.base, w->width, w->height, w->c->settings.color_format, + w->c->settings.color_space, VK_IMAGE_USAGE_TRANSFER_DST_BIT, + VK_PRESENT_MODE_MAILBOX_KHR); + + VkSemaphoreCreateInfo sem_info = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, + .pNext = NULL, + .flags = 0, + }; + + VkResult ret = vk->vkCreateSemaphore(vk->device, &sem_info, NULL, &w->acquire); + if (ret != VK_SUCCESS) { + COMP_ERROR(c, "vkCreateSemaphore: %s", vk_result_string(ret)); + } + + ret = vk->vkCreateSemaphore(vk->device, &sem_info, NULL, &w->submit); + if (ret != VK_SUCCESS) { + COMP_ERROR(c, "vkCreateSemaphore: %s", vk_result_string(ret)); + } + + os_thread_helper_init(&w->oth); + os_thread_helper_start(&w->oth, window_peek_run_thread, w); + + return w; +} + +void +comp_window_peek_destroy(struct comp_window_peek **w_ptr) +{ + struct comp_window_peek *w = *w_ptr; + if (w == NULL) { + return; + } + + w->running = false; + + SDL_DestroyWindow(w->window); + + comp_target_swapchain_cleanup(&w->base); + os_thread_helper_destroy(&w->oth); + + free(w); + + *w_ptr = NULL; +} + +void +comp_window_peek_blit(struct comp_window_peek *w, VkImage src, int32_t width, int32_t height) +{ + if (w->hidden) { + return; + } + + if (w->width != w->base.base.width || w->height != w->base.base.height) { + COMP_DEBUG(w->c, "Resizing swapchain"); + comp_target_create_images(&w->base.base, w->width, w->height, w->c->settings.color_format, + w->c->settings.color_space, VK_IMAGE_USAGE_TRANSFER_DST_BIT, + VK_PRESENT_MODE_MAILBOX_KHR); + } + + while (!comp_target_check_ready(&w->base.base)) + ; + + uint32_t current; + VkResult ret = comp_target_acquire(&w->base.base, w->acquire, ¤t); + if (ret != VK_SUCCESS) { + COMP_ERROR(w->c, "comp_target_acquire: %s", vk_result_string(ret)); + } + + VkImage dst = w->base.base.images[current].handle; + + struct vk_bundle *vk = get_vk(w); + + VkCommandBuffer cmd; + vk_init_cmd_buffer(vk, &cmd); + + // For submitting commands. + os_mutex_lock(&vk->cmd_pool_mutex); + + VkImageSubresourceRange range = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + }; + + // Barrier to make source a source + vk_cmd_image_barrier_locked( // + vk, // vk_bundle + cmd, // cmdbuffer + src, // image + VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, // srcAccessMask + VK_ACCESS_TRANSFER_READ_BIT, // dstAccessMask + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, // oldImageLayout + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, // newImageLayout + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, // srcStageMask + VK_PIPELINE_STAGE_TRANSFER_BIT, // dstStageMask + range); // subresourceRange + + // Barrier to make destination a destination + vk_cmd_image_barrier_locked( // + vk, // vk_bundle + cmd, // cmdbuffer + dst, // image + 0, // srcAccessMask + VK_ACCESS_TRANSFER_WRITE_BIT, // dstAccessMask + VK_IMAGE_LAYOUT_UNDEFINED, // oldImageLayout + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, // newImageLayout + VK_PIPELINE_STAGE_TRANSFER_BIT, // srcStageMask + VK_PIPELINE_STAGE_TRANSFER_BIT, // dstStageMask + range); // subresourceRange + + VkImageBlit blit = {.srcSubresource = + { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .layerCount = 1, + }, + .dstSubresource = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .layerCount = 1, + }}; + + blit.srcOffsets[1].x = width; + blit.srcOffsets[1].y = height; + blit.srcOffsets[1].z = 1; + + blit.dstOffsets[1].x = w->width; + blit.dstOffsets[1].y = w->height; + blit.dstOffsets[1].z = 1; + + vk->vkCmdBlitImage( // + cmd, // commandBuffer + src, // srcImage + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, // srcImageLayout + dst, // dstImage + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, // dstImageLayout + 1, // regionCount + &blit, // pRegions + VK_FILTER_LINEAR // filter + ); + + // Reset destination + vk_cmd_image_barrier_locked( // + vk, // vk_bundle + cmd, // cmdbuffer + dst, // image + VK_ACCESS_TRANSFER_WRITE_BIT, // srcAccessMask + 0, // dstAccessMask + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, // oldImageLayout + VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, // newImageLayout + VK_PIPELINE_STAGE_TRANSFER_BIT, // srcStageMask + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, // dstStageMask + range); // subresourceRange + + // Reset src + vk_cmd_image_barrier_locked( // + vk, // vk_bundle + cmd, // cmdbuffer + src, // image + VK_ACCESS_TRANSFER_READ_BIT, // srcAccessMask + 0, // dstAccessMask + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, // oldImageLayout + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, // newImageLayout + VK_PIPELINE_STAGE_TRANSFER_BIT, // srcStageMask + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // dstStageMask + range); // subresourceRange + + ret = vk->vkEndCommandBuffer(cmd); + + if (ret != VK_SUCCESS) { + VK_ERROR(vk, "Error: Could not end command buffer.\n"); + vk->vkFreeCommandBuffers(vk->device, vk->cmd_pool, 1, &cmd); + os_mutex_unlock(&vk->cmd_pool_mutex); + return; + } + + os_mutex_unlock(&vk->cmd_pool_mutex); + + VkPipelineStageFlags submit_flags = VK_PIPELINE_STAGE_TRANSFER_BIT; + + // Waits for command to finish. + VkSubmitInfo submit = { + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, + .pNext = NULL, + .waitSemaphoreCount = 1, + .pWaitSemaphores = &w->acquire, + .pWaitDstStageMask = &submit_flags, + .commandBufferCount = 1, + .pCommandBuffers = &cmd, + .signalSemaphoreCount = 1, + .pSignalSemaphores = &w->submit, + }; + + os_mutex_lock(&vk->queue_mutex); + os_mutex_lock(&vk->cmd_pool_mutex); + ret = vk->vkQueueSubmit(vk->queue, 1, &submit, VK_NULL_HANDLE); + os_mutex_unlock(&vk->cmd_pool_mutex); + os_mutex_unlock(&vk->queue_mutex); + + if (ret != VK_SUCCESS) { + VK_ERROR(vk, "Error: Could not submit to queue.\n"); + return; + } + + VkPresentInfoKHR present = { + .sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR, + .pNext = NULL, + .waitSemaphoreCount = 1, + .pWaitSemaphores = &w->submit, + .swapchainCount = 1, + .pSwapchains = &w->base.swapchain.handle, + .pImageIndices = ¤t, + .pResults = NULL, + }; + + os_mutex_lock(&vk->queue_mutex); + ret = vk->vkQueuePresentKHR(vk->queue, &present); + os_mutex_unlock(&vk->queue_mutex); + + if (ret != VK_SUCCESS) { + VK_ERROR(vk, "Error: could not present to queue.\n"); + return; + } +} diff --git a/src/xrt/compositor/main/comp_window_peek.h b/src/xrt/compositor/main/comp_window_peek.h new file mode 100644 index 000000000..47b772ca5 --- /dev/null +++ b/src/xrt/compositor/main/comp_window_peek.h @@ -0,0 +1,69 @@ +// Copyright 2022, Simon Zeni +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Displays the content of one or both eye onto a desktop window + * @author Simon Zeni + * @ingroup comp_main + */ + +#pragma once + +#include "xrt/xrt_config_have.h" + +#ifndef XRT_FEATURE_WINDOW_PEEK +#error "XRT_FEATURE_WINDOW_PEEK not enabled" +#endif + +#include "os/os_threading.h" + +#ifdef XRT_HAVE_SDL2 +#include +#else +#error "comp_window_peek.h requires SDL2" +#endif + +struct comp_compositor; +struct comp_renderer; + +#ifdef __cplusplus +extern "C" { +#endif + +enum comp_window_peek_eye +{ + COMP_WINDOW_PEEK_EYE_LEFT = 0, + COMP_WINDOW_PEEK_EYE_RIGHT = 1, + COMP_WINDOW_PEEK_EYE_BOTH = 2, +}; + +struct comp_window_peek +{ + struct comp_target_swapchain base; + struct comp_compositor *c; + + enum comp_window_peek_eye eye; + SDL_Window *window; + uint32_t width, height; + bool running; + bool hidden; + + VkSurfaceKHR surface; + VkSemaphore acquire; + VkSemaphore submit; + + struct os_thread_helper oth; +}; + +struct comp_window_peek * +comp_window_peek_create(struct comp_compositor *c); + +void +comp_window_peek_destroy(struct comp_window_peek **w_ptr); + +void +comp_window_peek_blit(struct comp_window_peek *w, VkImage src, int32_t width, int32_t height); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/include/xrt/xrt_config_build.h.cmake_in b/src/xrt/include/xrt/xrt_config_build.h.cmake_in index 1944fb63e..f12a501fc 100644 --- a/src/xrt/include/xrt/xrt_config_build.h.cmake_in +++ b/src/xrt/include/xrt/xrt_config_build.h.cmake_in @@ -27,3 +27,4 @@ #cmakedefine XRT_FEATURE_SLAM #cmakedefine XRT_FEATURE_TRACING #cmakedefine XRT_FEATURE_CLIENT_DEBUG_GUI +#cmakedefine XRT_FEATURE_WINDOW_PEEK