From 4c09d20f80843e6274e85c4a292549ca08bf464f Mon Sep 17 00:00:00 2001 From: bjorn Date: Thu, 26 May 2022 15:19:51 -0700 Subject: [PATCH] c/render: render_resources has its own command pool; Currently, there is a single command pool in the vk bundle, shared by everyone. Since command pools (and command buffers allocated from those pools) can only be used on one thread at a time, this requires locking. However, the main point of having these annoying command pool things in the first place is that you can use one for each thread/lifetime/area in the app and avoid the overhead of the locks (both computational and cognitive). In this change I have given the rendering bits of the compositor its own command pool. Instead of allocating and freeing a command buffer every frame, a single command buffer is allocated from the pool during initialization, and the pool is reset at the beginning of each frame. Normally, multiple pools would need to be used, but this is not necessary in monado because frames are serialized. The `TRANSIENT` and `ONE_TIME_SUBMIT` flags have been added, which can allow for some driver optimizations. The render code no longer takes out the command pool mutex. The shared command pool is still there for a few remaining places where vulkan work needs to be done outside the compositor. I used the command buffer vulkan helpers when possible, but I would maybe propose the idea of removing them, since they aren't really wrapping much at this point. The `C` macro helps a lot and it's a bit easier to see the Vulkan details in front of you instead of needing to switch back and forth between the helper. Later, I think it would be cool to apply and document some constraints like "the queue is only accessed in functions XYZ, the render_resources command pool must only be accessed in layer_commit from 1 thread" etc. --- src/xrt/auxiliary/vk/vk_command_buffer.c | 20 +++---- src/xrt/auxiliary/vk/vk_helpers.h | 10 ++-- src/xrt/compositor/main/comp_renderer.c | 8 +-- src/xrt/compositor/render/render_compute.c | 55 ++++++++----------- src/xrt/compositor/render/render_gfx.c | 56 +++++++------------- src/xrt/compositor/render/render_interface.h | 11 ++-- src/xrt/compositor/render/render_resources.c | 11 ++++ 7 files changed, 71 insertions(+), 100 deletions(-) diff --git a/src/xrt/auxiliary/vk/vk_command_buffer.c b/src/xrt/auxiliary/vk/vk_command_buffer.c index 16f2f2981..5eba3ac07 100644 --- a/src/xrt/auxiliary/vk/vk_command_buffer.c +++ b/src/xrt/auxiliary/vk/vk_command_buffer.c @@ -13,28 +13,24 @@ VkResult -vk_create_command_buffer(struct vk_bundle *vk, VkCommandBuffer *out_command_buffer) +vk_create_command_buffer(struct vk_bundle *vk, VkCommandPool pool, VkCommandBuffer *out_command_buffer) { VkResult ret; VkCommandBufferAllocateInfo cmd_buffer_info = { .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, - .commandPool = vk->cmd_pool, + .commandPool = pool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1, }; VkCommandBuffer cmd = VK_NULL_HANDLE; - os_mutex_lock(&vk->cmd_pool_mutex); - ret = vk->vkAllocateCommandBuffers( // vk->device, // device &cmd_buffer_info, // pAllocateInfo &cmd); // pCommandBuffers - os_mutex_unlock(&vk->cmd_pool_mutex); - if (ret != VK_SUCCESS) { VK_ERROR(vk, "vkCreateFramebuffer failed: %s", vk_result_string(ret)); return ret; @@ -46,17 +42,13 @@ vk_create_command_buffer(struct vk_bundle *vk, VkCommandBuffer *out_command_buff } void -vk_destroy_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer) +vk_destroy_command_buffer(struct vk_bundle *vk, VkCommandPool pool, VkCommandBuffer command_buffer) { - os_mutex_lock(&vk->cmd_pool_mutex); - vk->vkFreeCommandBuffers( // vk->device, // device - vk->cmd_pool, // commandPool + pool, // commandPool 1, // commandBufferCount &command_buffer); // pCommandBuffers - - os_mutex_unlock(&vk->cmd_pool_mutex); } VkResult @@ -66,6 +58,7 @@ vk_begin_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer) VkCommandBufferBeginInfo command_buffer_info = { .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, }; ret = vk->vkBeginCommandBuffer( // @@ -85,8 +78,7 @@ vk_end_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer) VkResult ret; // End the command buffer. - ret = vk->vkEndCommandBuffer( // - command_buffer); // commandBuffer + ret = vk->vkEndCommandBuffer(command_buffer); if (ret != VK_SUCCESS) { VK_ERROR(vk, "vkEndCommandBuffer failed: %s", vk_result_string(ret)); return ret; diff --git a/src/xrt/auxiliary/vk/vk_helpers.h b/src/xrt/auxiliary/vk/vk_helpers.h index 5f1520cb9..70a9ecacc 100644 --- a/src/xrt/auxiliary/vk/vk_helpers.h +++ b/src/xrt/auxiliary/vk/vk_helpers.h @@ -835,7 +835,7 @@ vk_update_buffer(struct vk_bundle *vk, float *buffer, size_t buffer_size, VkDevi /* * - * Helpers for writing command buffers. + * Helpers for writing command buffers using the global command pool. * */ @@ -1010,20 +1010,20 @@ vk_create_compute_pipeline(struct vk_bundle *vk, */ /*! - * Creates a new command buffer using the bundle's pool, takes the cmd_pool_mutex. + * Creates a new command buffer using a specified pool. * * Does error logging. */ VkResult -vk_create_command_buffer(struct vk_bundle *vk, VkCommandBuffer *out_command_buffer); +vk_create_command_buffer(struct vk_bundle *vk, VkCommandPool pool, VkCommandBuffer *out_command_buffer); /*! - * Destroys a command buffer, takes the cmd_pool_mutex. + * Destroys a command buffer. * * Does error logging. */ void -vk_destroy_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer); +vk_destroy_command_buffer(struct vk_bundle *vk, VkCommandPool pool, VkCommandBuffer command_buffer); /*! * Issues the vkBeginCommandBuffer function on the command buffer. diff --git a/src/xrt/compositor/main/comp_renderer.c b/src/xrt/compositor/main/comp_renderer.c index 4d7bff968..e40885f3d 100644 --- a/src/xrt/compositor/main/comp_renderer.c +++ b/src/xrt/compositor/main/comp_renderer.c @@ -882,7 +882,7 @@ dispatch_graphics(struct comp_renderer *r, struct render_gfx *rr) renderer_build_rendering(r, rr, rtr, src_samplers, src_image_views, src_norm_rects); - renderer_submit_queue(r, rr->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); + renderer_submit_queue(r, rr->r->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); return; } @@ -906,7 +906,7 @@ dispatch_graphics(struct comp_renderer *r, struct render_gfx *rr) do_gfx_mesh_and_proj(r, rr, rtr, layer, lvd, rvd); - renderer_submit_queue(r, rr->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); + renderer_submit_queue(r, rr->r->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); // We mark afterwards to not include CPU time spent. comp_target_mark_submit(ct, c->frame.rendering.id, os_monotonic_get_ns()); @@ -919,7 +919,7 @@ dispatch_graphics(struct comp_renderer *r, struct render_gfx *rr) do_gfx_mesh_and_proj(r, rr, rtr, layer, lvd, rvd); - renderer_submit_queue(r, rr->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); + renderer_submit_queue(r, rr->r->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); // We mark afterwards to not include CPU time spent. comp_target_mark_submit(ct, c->frame.rendering.id, os_monotonic_get_ns()); @@ -1098,7 +1098,7 @@ dispatch_compute(struct comp_renderer *r, struct render_compute *crc) comp_target_mark_submit(ct, c->frame.rendering.id, os_monotonic_get_ns()); - renderer_submit_queue(r, crc->cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); + renderer_submit_queue(r, crc->r->cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); } diff --git a/src/xrt/compositor/render/render_compute.c b/src/xrt/compositor/render/render_compute.c index 940d5c6f4..70afcb075 100644 --- a/src/xrt/compositor/render/render_compute.c +++ b/src/xrt/compositor/render/render_compute.c @@ -262,8 +262,6 @@ render_compute_init(struct render_compute *crc, struct render_resources *r) struct vk_bundle *vk = r->vk; crc->r = r; - C(vk_create_command_buffer(vk, &crc->cmd)); - C(vk_create_descriptor_set( // vk, // r->compute.descriptor_pool, // descriptor_pool @@ -278,26 +276,21 @@ render_compute_begin(struct render_compute *crc) { struct vk_bundle *vk = vk_from_crc(crc); - os_mutex_lock(&vk->cmd_pool_mutex); - VkResult ret = vk_begin_command_buffer(vk, crc->cmd); - if (ret != VK_SUCCESS) { - os_mutex_unlock(&vk->cmd_pool_mutex); - return false; - } + C(vk->vkResetCommandPool(vk->device, crc->r->cmd_pool, 0)); + C(vk_begin_command_buffer(vk, crc->r->cmd)); vk->vkCmdResetQueryPool( // - crc->cmd, // commandBuffer + crc->r->cmd, // commandBuffer crc->r->query_pool, // queryPool 0, // firstQuery 2); // queryCount vk->vkCmdWriteTimestamp( // - crc->cmd, // commandBuffer + crc->r->cmd, // commandBuffer VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // pipelineStage crc->r->query_pool, // queryPool 0); // query - // Yes we leave the mutex locked. return true; } @@ -307,16 +300,12 @@ render_compute_end(struct render_compute *crc) struct vk_bundle *vk = vk_from_crc(crc); vk->vkCmdWriteTimestamp( // - crc->cmd, // commandBuffer + crc->r->cmd, // commandBuffer VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, // pipelineStage crc->r->query_pool, // queryPool 1); // query - VkResult ret = vk_end_command_buffer(vk, crc->cmd); - os_mutex_unlock(&vk->cmd_pool_mutex); - if (ret != VK_SUCCESS) { - return false; - } + C(vk_end_command_buffer(vk, crc->r->cmd)); return true; } @@ -328,8 +317,6 @@ render_compute_close(struct render_compute *crc) struct vk_bundle *vk = vk_from_crc(crc); - vk_destroy_command_buffer(vk, crc->cmd); - // Reclaimed by vkResetDescriptorPool. crc->descriptor_set = VK_NULL_HANDLE; @@ -398,7 +385,7 @@ render_compute_projection_timewarp(struct render_compute *crc, vk_cmd_image_barrier_gpu_locked( // vk, // - crc->cmd, // + r->cmd, // target_image, // 0, // VK_ACCESS_SHADER_WRITE_BIT, // @@ -427,12 +414,12 @@ render_compute_projection_timewarp(struct render_compute *crc, crc->descriptor_set); // vk->vkCmdBindPipeline( // - crc->cmd, // commandBuffer + r->cmd, // commandBuffer VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint r->compute.distortion_timewarp_pipeline); // pipeline vk->vkCmdBindDescriptorSets( // - crc->cmd, // commandBuffer + r->cmd, // commandBuffer VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint r->compute.pipeline_layout, // layout 0, // firstSet @@ -447,7 +434,7 @@ render_compute_projection_timewarp(struct render_compute *crc, assert(w != 0 && h != 0); vk->vkCmdDispatch( // - crc->cmd, // commandBuffer + r->cmd, // commandBuffer w, // groupCountX h, // groupCountY 2); // groupCountZ @@ -465,7 +452,7 @@ render_compute_projection_timewarp(struct render_compute *crc, }; vk->vkCmdPipelineBarrier( // - crc->cmd, // + r->cmd, // VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // 0, // @@ -518,7 +505,7 @@ render_compute_projection(struct render_compute *crc, vk_cmd_image_barrier_gpu_locked( // vk, // - crc->cmd, // + r->cmd, // target_image, // 0, // VK_ACCESS_SHADER_WRITE_BIT, // @@ -547,12 +534,12 @@ render_compute_projection(struct render_compute *crc, crc->descriptor_set); // vk->vkCmdBindPipeline( // - crc->cmd, // commandBuffer + r->cmd, // commandBuffer VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint r->compute.distortion_pipeline); // pipeline vk->vkCmdBindDescriptorSets( // - crc->cmd, // commandBuffer + r->cmd, // commandBuffer VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint r->compute.pipeline_layout, // layout 0, // firstSet @@ -567,7 +554,7 @@ render_compute_projection(struct render_compute *crc, assert(w != 0 && h != 0); vk->vkCmdDispatch( // - crc->cmd, // commandBuffer + r->cmd, // commandBuffer w, // groupCountX h, // groupCountY 2); // groupCountZ @@ -585,7 +572,7 @@ render_compute_projection(struct render_compute *crc, }; vk->vkCmdPipelineBarrier( // - crc->cmd, // + r->cmd, // VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // 0, // @@ -639,7 +626,7 @@ render_compute_clear(struct render_compute *crc, // vk_cmd_image_barrier_gpu_locked( // vk, // - crc->cmd, // + r->cmd, // target_image, // 0, // VK_ACCESS_SHADER_WRITE_BIT, // @@ -668,12 +655,12 @@ render_compute_clear(struct render_compute *crc, // crc->descriptor_set); // descriptor_set vk->vkCmdBindPipeline( // - crc->cmd, // commandBuffer + r->cmd, // commandBuffer VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint r->compute.clear_pipeline); // pipeline vk->vkCmdBindDescriptorSets( // - crc->cmd, // commandBuffer + r->cmd, // commandBuffer VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint r->compute.pipeline_layout, // layout 0, // firstSet @@ -688,7 +675,7 @@ render_compute_clear(struct render_compute *crc, // assert(w != 0 && h != 0); vk->vkCmdDispatch( // - crc->cmd, // commandBuffer + r->cmd, // commandBuffer w, // groupCountX h, // groupCountY 2); // groupCountZ @@ -706,7 +693,7 @@ render_compute_clear(struct render_compute *crc, // }; vk->vkCmdPipelineBarrier( // - crc->cmd, // + r->cmd, // VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // 0, // diff --git a/src/xrt/compositor/render/render_gfx.c b/src/xrt/compositor/render/render_gfx.c index cce5cbc8a..647790882 100644 --- a/src/xrt/compositor/render/render_gfx.c +++ b/src/xrt/compositor/render/render_gfx.c @@ -498,12 +498,6 @@ render_gfx_init(struct render_gfx *rr, struct render_resources *r) rr->r = r; - /* - * Per rendering. - */ - - C(vk_create_command_buffer(vk, &rr->cmd)); - /* * Mesh per view */ @@ -528,26 +522,21 @@ render_gfx_begin(struct render_gfx *rr) { struct vk_bundle *vk = vk_from_rr(rr); - os_mutex_lock(&vk->cmd_pool_mutex); - VkResult ret = vk_begin_command_buffer(vk, rr->cmd); - if (ret != VK_SUCCESS) { - os_mutex_unlock(&vk->cmd_pool_mutex); - return false; - } + C(vk->vkResetCommandPool(vk->device, rr->r->cmd_pool, 0)); + C(vk_begin_command_buffer(vk, rr->r->cmd)); vk->vkCmdResetQueryPool( // - rr->cmd, // commandBuffer + rr->r->cmd, // commandBuffer rr->r->query_pool, // queryPool 0, // firstQuery 2); // queryCount vk->vkCmdWriteTimestamp( // - rr->cmd, // commandBuffer + rr->r->cmd, // commandBuffer VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // pipelineStage rr->r->query_pool, // queryPool 0); // query - // Yes we leave the mutex locked. return true; } @@ -557,16 +546,12 @@ render_gfx_end(struct render_gfx *rr) struct vk_bundle *vk = vk_from_rr(rr); vk->vkCmdWriteTimestamp( // - rr->cmd, // commandBuffer + rr->r->cmd, // commandBuffer VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, // pipelineStage rr->r->query_pool, // queryPool 1); // query - VkResult ret = vk_end_command_buffer(vk, rr->cmd); - os_mutex_unlock(&vk->cmd_pool_mutex); - if (ret != VK_SUCCESS) { - return false; - } + C(vk_end_command_buffer(vk, rr->r->cmd)); return true; } @@ -577,9 +562,6 @@ render_gfx_close(struct render_gfx *rr) struct vk_bundle *vk = vk_from_rr(rr); struct render_resources *r = rr->r; - vk_destroy_command_buffer(vk, rr->cmd); - rr->cmd = VK_NULL_HANDLE; - // Reclaimed by vkResetDescriptorPool. rr->views[0].mesh.descriptor_set = VK_NULL_HANDLE; rr->views[1].mesh.descriptor_set = VK_NULL_HANDLE; @@ -609,7 +591,7 @@ render_gfx_begin_target(struct render_gfx *rr, struct render_gfx_target_resource // This is shared across both views. begin_render_pass(vk, // - rr->cmd, // + rr->r->cmd, // rr->rtr->render_pass, // rr->rtr->framebuffer, // rr->rtr->data.width, // @@ -627,7 +609,7 @@ render_gfx_end_target(struct render_gfx *rr) rr->rtr = NULL; // Stop the [shared] render pass. - vk->vkCmdEndRenderPass(rr->cmd); + vk->vkCmdEndRenderPass(rr->r->cmd); } void @@ -655,7 +637,7 @@ render_gfx_begin_view(struct render_gfx *rr, uint32_t view, struct render_viewpo .maxDepth = 1.0f, }; - vk->vkCmdSetViewport(rr->cmd, // commandBuffer + vk->vkCmdSetViewport(rr->r->cmd, // commandBuffer 0, // firstViewport 1, // viewportCount &viewport); // pViewports @@ -677,10 +659,10 @@ render_gfx_begin_view(struct render_gfx *rr, uint32_t view, struct render_viewpo }, }; - vk->vkCmdSetScissor(rr->cmd, // commandBuffer - 0, // firstScissor - 1, // scissorCount - &scissor); // pScissors + vk->vkCmdSetScissor(rr->r->cmd, // commandBuffer + 0, // firstScissor + 1, // scissorCount + &scissor); // pScissors } void @@ -705,7 +687,7 @@ render_gfx_distortion(struct render_gfx *rr) VkDescriptorSet descriptor_sets[1] = {v->mesh.descriptor_set}; vk->vkCmdBindDescriptorSets( // - rr->cmd, // commandBuffer + r->cmd, // commandBuffer VK_PIPELINE_BIND_POINT_GRAPHICS, // pipelineBindPoint r->mesh.pipeline_layout, // layout 0, // firstSet @@ -715,7 +697,7 @@ render_gfx_distortion(struct render_gfx *rr) NULL); // pDynamicOffsets vk->vkCmdBindPipeline( // - rr->cmd, // commandBuffer + r->cmd, // commandBuffer VK_PIPELINE_BIND_POINT_GRAPHICS, // pipelineBindPoint rr->rtr->mesh.pipeline); // pipeline @@ -729,7 +711,7 @@ render_gfx_distortion(struct render_gfx *rr) assert(ARRAY_SIZE(buffers) == ARRAY_SIZE(offsets)); vk->vkCmdBindVertexBuffers( // - rr->cmd, // commandBuffer + r->cmd, // commandBuffer 0, // firstBinding ARRAY_SIZE(buffers), // bindingCount buffers, // pBuffers @@ -742,13 +724,13 @@ render_gfx_distortion(struct render_gfx *rr) if (r->mesh.index_count_total > 0) { vk->vkCmdBindIndexBuffer( // - rr->cmd, // commandBuffer + r->cmd, // commandBuffer r->mesh.ibo.buffer, // buffer 0, // offset VK_INDEX_TYPE_UINT32); // indexType vk->vkCmdDrawIndexed( // - rr->cmd, // commandBuffer + r->cmd, // commandBuffer r->mesh.index_counts[view], // indexCount 1, // instanceCount r->mesh.index_offsets[view], // firstIndex @@ -756,7 +738,7 @@ render_gfx_distortion(struct render_gfx *rr) 0); // firstInstance } else { vk->vkCmdDraw( // - rr->cmd, // commandBuffer + r->cmd, // commandBuffer r->mesh.vertex_count, // vertexCount 1, // instanceCount 0, // firstVertex diff --git a/src/xrt/compositor/render/render_interface.h b/src/xrt/compositor/render/render_interface.h index 8a73c49db..c535d9081 100644 --- a/src/xrt/compositor/render/render_interface.h +++ b/src/xrt/compositor/render/render_interface.h @@ -212,6 +212,8 @@ struct render_resources //! Shared for all rendering. VkPipelineCache pipeline_cache; + VkCommandPool cmd_pool; + VkQueryPool query_pool; @@ -219,6 +221,9 @@ struct render_resources * Static */ + //! Command buffer for recording everything. + VkCommandBuffer cmd; + struct { //! The binding index for the source texture. @@ -478,9 +483,6 @@ struct render_gfx //! The current target we are rendering too, can change during command building. struct render_gfx_target_resources *rtr; - //! Command buffer where all commands are recorded. - VkCommandBuffer cmd; - //! Holds per view data. struct render_gfx_view views[2]; @@ -619,9 +621,6 @@ struct render_compute //! Shared resources. struct render_resources *r; - //! Command buffer where all commands are recorded. - VkCommandBuffer cmd; - //! Shared descriptor set between clear, projection and timewarp. VkDescriptorSet descriptor_set; }; diff --git a/src/xrt/compositor/render/render_resources.c b/src/xrt/compositor/render/render_resources.c index 9128686df..51901aec5 100644 --- a/src/xrt/compositor/render/render_resources.c +++ b/src/xrt/compositor/render/render_resources.c @@ -610,6 +610,16 @@ render_resources_init(struct render_resources *r, C(vk_create_pipeline_cache(vk, &r->pipeline_cache)); + VkCommandPoolCreateInfo command_pool_info = { + .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, + .flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT, + .queueFamilyIndex = vk->queue_family_index, + }; + + C(vk->vkCreateCommandPool(vk->device, &command_pool_info, NULL, &r->cmd_pool)); + + C(vk_create_command_buffer(vk, r->cmd_pool, &r->cmd)); + /* * Mesh static. @@ -823,6 +833,7 @@ render_resources_close(struct render_resources *r) D(DescriptorSetLayout, r->mesh.descriptor_set_layout); D(PipelineLayout, r->mesh.pipeline_layout); D(PipelineCache, r->pipeline_cache); + D(CommandPool, r->cmd_pool); D(DescriptorPool, r->mesh.descriptor_pool); D(QueryPool, r->query_pool); render_buffer_close(vk, &r->mesh.vbo);