mirrors/monado
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/monado/monado.git synced 2025-01-17 04:15:44 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

c/render: render_resources has its own command pool;

Browse source 
Currently, there is a single command pool in the vk bundle, shared by
everyone.  Since command pools (and command buffers allocated from those
pools) can only be used on one thread at a time, this requires locking.
However, the main point of having these annoying command pool things in
the first place is that you can use one for each thread/lifetime/area in
the app and avoid the overhead of the locks (both computational and
cognitive).

In this change I have given the rendering bits of the compositor its own
command pool.  Instead of allocating and freeing a command buffer every
frame, a single command buffer is allocated from the pool during
initialization, and the pool is reset at the beginning of each frame.
Normally, multiple pools would need to be used, but this is not
necessary in monado because frames are serialized. The `TRANSIENT` and
`ONE_TIME_SUBMIT` flags have been added, which can allow for some driver
optimizations.  The render code no longer takes out the command pool
mutex.  The shared command pool is still there for a few remaining
places where vulkan work needs to be done outside the compositor.

I used the command buffer vulkan helpers when possible, but I would
maybe propose the idea of removing them, since they aren't really
wrapping much at this point.  The `C` macro helps a lot and it's a bit
easier to see the Vulkan details in front of you instead of needing to
switch back and forth between the helper.

Later, I think it would be cool to apply and document some constraints
like "the queue is only accessed in functions XYZ, the render_resources
command pool must only be accessed in layer_commit from 1 thread" etc.
This commit is contained in:
bjorn 2022-05-26 15:19:51 -07:00 committed by Jakob Bornecrantz
parent 24c9dabfbb
commit 4c09d20f80
7 changed files with 71 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/xrt/auxiliary/vk/vk_command_buffer.c
View file

@ -13,28 +13,24 @@
VkResult
vk_create_command_buffer(struct vk_bundle *vk, VkCommandBuffer *out_command_buffer)
vk_create_command_buffer(struct vk_bundle *vk, VkCommandPool pool, VkCommandBuffer *out_command_buffer)
{
VkResult ret;
VkCommandBufferAllocateInfo cmd_buffer_info = {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
.commandPool = vk->cmd_pool,
.commandPool = pool,
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
.commandBufferCount = 1,
};
VkCommandBuffer cmd = VK_NULL_HANDLE;
os_mutex_lock(&vk->cmd_pool_mutex);
ret = vk->vkAllocateCommandBuffers( //
vk->device, // device
&cmd_buffer_info, // pAllocateInfo
&cmd); // pCommandBuffers
os_mutex_unlock(&vk->cmd_pool_mutex);
if (ret != VK_SUCCESS) {
VK_ERROR(vk, "vkCreateFramebuffer failed: %s", vk_result_string(ret));
return ret;
@ -46,17 +42,13 @@ vk_create_command_buffer(struct vk_bundle *vk, VkCommandBuffer *out_command_buff
}
void
vk_destroy_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer)
vk_destroy_command_buffer(struct vk_bundle *vk, VkCommandPool pool, VkCommandBuffer command_buffer)
{
os_mutex_lock(&vk->cmd_pool_mutex);
vk->vkFreeCommandBuffers( //
vk->device, // device
vk->cmd_pool, // commandPool
pool, // commandPool
1, // commandBufferCount
&command_buffer); // pCommandBuffers
os_mutex_unlock(&vk->cmd_pool_mutex);
}
VkResult
@ -66,6 +58,7 @@ vk_begin_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer)
VkCommandBufferBeginInfo command_buffer_info = {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
};
ret = vk->vkBeginCommandBuffer( //
@ -85,8 +78,7 @@ vk_end_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer)
VkResult ret;
// End the command buffer.
ret = vk->vkEndCommandBuffer( //
command_buffer); // commandBuffer
ret = vk->vkEndCommandBuffer(command_buffer);
if (ret != VK_SUCCESS) {
VK_ERROR(vk, "vkEndCommandBuffer failed: %s", vk_result_string(ret));
return ret;

10
src/xrt/auxiliary/vk/vk_helpers.h
View file

@ -835,7 +835,7 @@ vk_update_buffer(struct vk_bundle *vk, float *buffer, size_t buffer_size, VkDevi
/*
*
* Helpers for writing command buffers.
* Helpers for writing command buffers using the global command pool.
*
*/
@ -1010,20 +1010,20 @@ vk_create_compute_pipeline(struct vk_bundle *vk,
*/
/*!
* Creates a new command buffer using the bundle's pool, takes the cmd_pool_mutex.
* Creates a new command buffer using a specified pool.
*
* Does error logging.
*/
VkResult
vk_create_command_buffer(struct vk_bundle *vk, VkCommandBuffer *out_command_buffer);
vk_create_command_buffer(struct vk_bundle *vk, VkCommandPool pool, VkCommandBuffer *out_command_buffer);
/*!
* Destroys a command buffer, takes the cmd_pool_mutex.
* Destroys a command buffer.
*
* Does error logging.
*/
void
vk_destroy_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer);
vk_destroy_command_buffer(struct vk_bundle *vk, VkCommandPool pool, VkCommandBuffer command_buffer);
/*!
* Issues the vkBeginCommandBuffer function on the command buffer.

8
src/xrt/compositor/main/comp_renderer.c
View file

@ -882,7 +882,7 @@ dispatch_graphics(struct comp_renderer *r, struct render_gfx *rr)
renderer_build_rendering(r, rr, rtr, src_samplers, src_image_views, src_norm_rects);
renderer_submit_queue(r, rr->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
renderer_submit_queue(r, rr->r->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
return;
}
@ -906,7 +906,7 @@ dispatch_graphics(struct comp_renderer *r, struct render_gfx *rr)
do_gfx_mesh_and_proj(r, rr, rtr, layer, lvd, rvd);
renderer_submit_queue(r, rr->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
renderer_submit_queue(r, rr->r->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
// We mark afterwards to not include CPU time spent.
comp_target_mark_submit(ct, c->frame.rendering.id, os_monotonic_get_ns());
@ -919,7 +919,7 @@ dispatch_graphics(struct comp_renderer *r, struct render_gfx *rr)
do_gfx_mesh_and_proj(r, rr, rtr, layer, lvd, rvd);
renderer_submit_queue(r, rr->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
renderer_submit_queue(r, rr->r->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
// We mark afterwards to not include CPU time spent.
comp_target_mark_submit(ct, c->frame.rendering.id, os_monotonic_get_ns());
@ -1098,7 +1098,7 @@ dispatch_compute(struct comp_renderer *r, struct render_compute *crc)
comp_target_mark_submit(ct, c->frame.rendering.id, os_monotonic_get_ns());
renderer_submit_queue(r, crc->cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
renderer_submit_queue(r, crc->r->cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
}

55
src/xrt/compositor/render/render_compute.c
View file

@ -262,8 +262,6 @@ render_compute_init(struct render_compute *crc, struct render_resources *r)
struct vk_bundle *vk = r->vk;
crc->r = r;
C(vk_create_command_buffer(vk, &crc->cmd));
C(vk_create_descriptor_set( //
vk, //
r->compute.descriptor_pool, // descriptor_pool
@ -278,26 +276,21 @@ render_compute_begin(struct render_compute *crc)
{
struct vk_bundle *vk = vk_from_crc(crc);
os_mutex_lock(&vk->cmd_pool_mutex);
VkResult ret = vk_begin_command_buffer(vk, crc->cmd);
if (ret != VK_SUCCESS) {
os_mutex_unlock(&vk->cmd_pool_mutex);
return false;
}
C(vk->vkResetCommandPool(vk->device, crc->r->cmd_pool, 0));
C(vk_begin_command_buffer(vk, crc->r->cmd));
vk->vkCmdResetQueryPool( //
crc->cmd, // commandBuffer
crc->r->cmd, // commandBuffer
crc->r->query_pool, // queryPool
0, // firstQuery
2); // queryCount
vk->vkCmdWriteTimestamp( //
crc->cmd, // commandBuffer
crc->r->cmd, // commandBuffer
VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // pipelineStage
crc->r->query_pool, // queryPool
0); // query
// Yes we leave the mutex locked.
return true;
}
@ -307,16 +300,12 @@ render_compute_end(struct render_compute *crc)
struct vk_bundle *vk = vk_from_crc(crc);
vk->vkCmdWriteTimestamp( //
crc->cmd, // commandBuffer
crc->r->cmd, // commandBuffer
VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, // pipelineStage
crc->r->query_pool, // queryPool
1); // query
VkResult ret = vk_end_command_buffer(vk, crc->cmd);
os_mutex_unlock(&vk->cmd_pool_mutex);
if (ret != VK_SUCCESS) {
return false;
}
C(vk_end_command_buffer(vk, crc->r->cmd));
return true;
}
@ -328,8 +317,6 @@ render_compute_close(struct render_compute *crc)
struct vk_bundle *vk = vk_from_crc(crc);
vk_destroy_command_buffer(vk, crc->cmd);
// Reclaimed by vkResetDescriptorPool.
crc->descriptor_set = VK_NULL_HANDLE;
@ -398,7 +385,7 @@ render_compute_projection_timewarp(struct render_compute *crc,
vk_cmd_image_barrier_gpu_locked( //
vk, //
crc->cmd, //
r->cmd, //
target_image, //
0, //
VK_ACCESS_SHADER_WRITE_BIT, //
@ -427,12 +414,12 @@ render_compute_projection_timewarp(struct render_compute *crc,
crc->descriptor_set); //
vk->vkCmdBindPipeline( //
crc->cmd, // commandBuffer
r->cmd, // commandBuffer
VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint
r->compute.distortion_timewarp_pipeline); // pipeline
vk->vkCmdBindDescriptorSets( //
crc->cmd, // commandBuffer
r->cmd, // commandBuffer
VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint
r->compute.pipeline_layout, // layout
0, // firstSet
@ -447,7 +434,7 @@ render_compute_projection_timewarp(struct render_compute *crc,
assert(w != 0 && h != 0);
vk->vkCmdDispatch( //
crc->cmd, // commandBuffer
r->cmd, // commandBuffer
w, // groupCountX
h, // groupCountY
2); // groupCountZ
@ -465,7 +452,7 @@ render_compute_projection_timewarp(struct render_compute *crc,
};
vk->vkCmdPipelineBarrier( //
crc->cmd, //
r->cmd, //
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, //
VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, //
0, //
@ -518,7 +505,7 @@ render_compute_projection(struct render_compute *crc,
vk_cmd_image_barrier_gpu_locked( //
vk, //
crc->cmd, //
r->cmd, //
target_image, //
0, //
VK_ACCESS_SHADER_WRITE_BIT, //
@ -547,12 +534,12 @@ render_compute_projection(struct render_compute *crc,
crc->descriptor_set); //
vk->vkCmdBindPipeline( //
crc->cmd, // commandBuffer
r->cmd, // commandBuffer
VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint
r->compute.distortion_pipeline); // pipeline
vk->vkCmdBindDescriptorSets( //
crc->cmd, // commandBuffer
r->cmd, // commandBuffer
VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint
r->compute.pipeline_layout, // layout
0, // firstSet
@ -567,7 +554,7 @@ render_compute_projection(struct render_compute *crc,
assert(w != 0 && h != 0);
vk->vkCmdDispatch( //
crc->cmd, // commandBuffer
r->cmd, // commandBuffer
w, // groupCountX
h, // groupCountY
2); // groupCountZ
@ -585,7 +572,7 @@ render_compute_projection(struct render_compute *crc,
};
vk->vkCmdPipelineBarrier( //
crc->cmd, //
r->cmd, //
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, //
VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, //
0, //
@ -639,7 +626,7 @@ render_compute_clear(struct render_compute *crc, //
vk_cmd_image_barrier_gpu_locked( //
vk, //
crc->cmd, //
r->cmd, //
target_image, //
0, //
VK_ACCESS_SHADER_WRITE_BIT, //
@ -668,12 +655,12 @@ render_compute_clear(struct render_compute *crc, //
crc->descriptor_set); // descriptor_set
vk->vkCmdBindPipeline( //
crc->cmd, // commandBuffer
r->cmd, // commandBuffer
VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint
r->compute.clear_pipeline); // pipeline
vk->vkCmdBindDescriptorSets( //
crc->cmd, // commandBuffer
r->cmd, // commandBuffer
VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint
r->compute.pipeline_layout, // layout
0, // firstSet
@ -688,7 +675,7 @@ render_compute_clear(struct render_compute *crc, //
assert(w != 0 && h != 0);
vk->vkCmdDispatch( //
crc->cmd, // commandBuffer
r->cmd, // commandBuffer
w, // groupCountX
h, // groupCountY
2); // groupCountZ
@ -706,7 +693,7 @@ render_compute_clear(struct render_compute *crc, //
};
vk->vkCmdPipelineBarrier( //
crc->cmd, //
r->cmd, //
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, //
VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, //
0, //

56
src/xrt/compositor/render/render_gfx.c
View file

@ -498,12 +498,6 @@ render_gfx_init(struct render_gfx *rr, struct render_resources *r)
rr->r = r;
/*
* Per rendering.
*/
C(vk_create_command_buffer(vk, &rr->cmd));
/*
* Mesh per view
*/
@ -528,26 +522,21 @@ render_gfx_begin(struct render_gfx *rr)
{
struct vk_bundle *vk = vk_from_rr(rr);
os_mutex_lock(&vk->cmd_pool_mutex);
VkResult ret = vk_begin_command_buffer(vk, rr->cmd);
if (ret != VK_SUCCESS) {
os_mutex_unlock(&vk->cmd_pool_mutex);
return false;
}
C(vk->vkResetCommandPool(vk->device, rr->r->cmd_pool, 0));
C(vk_begin_command_buffer(vk, rr->r->cmd));
vk->vkCmdResetQueryPool( //
rr->cmd, // commandBuffer
rr->r->cmd, // commandBuffer
rr->r->query_pool, // queryPool
0, // firstQuery
2); // queryCount
vk->vkCmdWriteTimestamp( //
rr->cmd, // commandBuffer
rr->r->cmd, // commandBuffer
VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // pipelineStage
rr->r->query_pool, // queryPool
0); // query
// Yes we leave the mutex locked.
return true;
}
@ -557,16 +546,12 @@ render_gfx_end(struct render_gfx *rr)
struct vk_bundle *vk = vk_from_rr(rr);
vk->vkCmdWriteTimestamp( //
rr->cmd, // commandBuffer
rr->r->cmd, // commandBuffer
VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, // pipelineStage
rr->r->query_pool, // queryPool
1); // query
VkResult ret = vk_end_command_buffer(vk, rr->cmd);
os_mutex_unlock(&vk->cmd_pool_mutex);
if (ret != VK_SUCCESS) {
return false;
}
C(vk_end_command_buffer(vk, rr->r->cmd));
return true;
}
@ -577,9 +562,6 @@ render_gfx_close(struct render_gfx *rr)
struct vk_bundle *vk = vk_from_rr(rr);
struct render_resources *r = rr->r;
vk_destroy_command_buffer(vk, rr->cmd);
rr->cmd = VK_NULL_HANDLE;
// Reclaimed by vkResetDescriptorPool.
rr->views[0].mesh.descriptor_set = VK_NULL_HANDLE;
rr->views[1].mesh.descriptor_set = VK_NULL_HANDLE;
@ -609,7 +591,7 @@ render_gfx_begin_target(struct render_gfx *rr, struct render_gfx_target_resource
// This is shared across both views.
begin_render_pass(vk, //
rr->cmd, //
rr->r->cmd, //
rr->rtr->render_pass, //
rr->rtr->framebuffer, //
rr->rtr->data.width, //
@ -627,7 +609,7 @@ render_gfx_end_target(struct render_gfx *rr)
rr->rtr = NULL;
// Stop the [shared] render pass.
vk->vkCmdEndRenderPass(rr->cmd);
vk->vkCmdEndRenderPass(rr->r->cmd);
}
void
@ -655,7 +637,7 @@ render_gfx_begin_view(struct render_gfx *rr, uint32_t view, struct render_viewpo
.maxDepth = 1.0f,
};
vk->vkCmdSetViewport(rr->cmd, // commandBuffer
vk->vkCmdSetViewport(rr->r->cmd, // commandBuffer
0, // firstViewport
1, // viewportCount
&viewport); // pViewports
@ -677,10 +659,10 @@ render_gfx_begin_view(struct render_gfx *rr, uint32_t view, struct render_viewpo
},
};
vk->vkCmdSetScissor(rr->cmd, // commandBuffer
0, // firstScissor
1, // scissorCount
&scissor); // pScissors
vk->vkCmdSetScissor(rr->r->cmd, // commandBuffer
0, // firstScissor
1, // scissorCount
&scissor); // pScissors
}
void
@ -705,7 +687,7 @@ render_gfx_distortion(struct render_gfx *rr)
VkDescriptorSet descriptor_sets[1] = {v->mesh.descriptor_set};
vk->vkCmdBindDescriptorSets( //
rr->cmd, // commandBuffer
r->cmd, // commandBuffer
VK_PIPELINE_BIND_POINT_GRAPHICS, // pipelineBindPoint
r->mesh.pipeline_layout, // layout
0, // firstSet
@ -715,7 +697,7 @@ render_gfx_distortion(struct render_gfx *rr)
NULL); // pDynamicOffsets
vk->vkCmdBindPipeline( //
rr->cmd, // commandBuffer
r->cmd, // commandBuffer
VK_PIPELINE_BIND_POINT_GRAPHICS, // pipelineBindPoint
rr->rtr->mesh.pipeline); // pipeline
@ -729,7 +711,7 @@ render_gfx_distortion(struct render_gfx *rr)
assert(ARRAY_SIZE(buffers) == ARRAY_SIZE(offsets));
vk->vkCmdBindVertexBuffers( //
rr->cmd, // commandBuffer
r->cmd, // commandBuffer
0, // firstBinding
ARRAY_SIZE(buffers), // bindingCount
buffers, // pBuffers
@ -742,13 +724,13 @@ render_gfx_distortion(struct render_gfx *rr)
if (r->mesh.index_count_total > 0) {
vk->vkCmdBindIndexBuffer( //
rr->cmd, // commandBuffer
r->cmd, // commandBuffer
r->mesh.ibo.buffer, // buffer
0, // offset
VK_INDEX_TYPE_UINT32); // indexType
vk->vkCmdDrawIndexed( //
rr->cmd, // commandBuffer
r->cmd, // commandBuffer
r->mesh.index_counts[view], // indexCount
1, // instanceCount
r->mesh.index_offsets[view], // firstIndex
@ -756,7 +738,7 @@ render_gfx_distortion(struct render_gfx *rr)
0); // firstInstance
} else {
vk->vkCmdDraw( //
rr->cmd, // commandBuffer
r->cmd, // commandBuffer
r->mesh.vertex_count, // vertexCount
1, // instanceCount
0, // firstVertex

11
src/xrt/compositor/render/render_interface.h
View file

@ -212,6 +212,8 @@ struct render_resources
//! Shared for all rendering.
VkPipelineCache pipeline_cache;
VkCommandPool cmd_pool;
VkQueryPool query_pool;
@ -219,6 +221,9 @@ struct render_resources
* Static
*/
//! Command buffer for recording everything.
VkCommandBuffer cmd;
struct
{
//! The binding index for the source texture.
@ -478,9 +483,6 @@ struct render_gfx
//! The current target we are rendering too, can change during command building.
struct render_gfx_target_resources *rtr;
//! Command buffer where all commands are recorded.
VkCommandBuffer cmd;
//! Holds per view data.
struct render_gfx_view views[2];
@ -619,9 +621,6 @@ struct render_compute
//! Shared resources.
struct render_resources *r;
//! Command buffer where all commands are recorded.
VkCommandBuffer cmd;
//! Shared descriptor set between clear, projection and timewarp.
VkDescriptorSet descriptor_set;
};

11
src/xrt/compositor/render/render_resources.c
View file

@ -610,6 +610,16 @@ render_resources_init(struct render_resources *r,
C(vk_create_pipeline_cache(vk, &r->pipeline_cache));
VkCommandPoolCreateInfo command_pool_info = {
.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT,
.queueFamilyIndex = vk->queue_family_index,
};
C(vk->vkCreateCommandPool(vk->device, &command_pool_info, NULL, &r->cmd_pool));
C(vk_create_command_buffer(vk, r->cmd_pool, &r->cmd));
/*
* Mesh static.
@ -823,6 +833,7 @@ render_resources_close(struct render_resources *r)
D(DescriptorSetLayout, r->mesh.descriptor_set_layout);
D(PipelineLayout, r->mesh.pipeline_layout);
D(PipelineCache, r->pipeline_cache);
D(CommandPool, r->cmd_pool);
D(DescriptorPool, r->mesh.descriptor_pool);
D(QueryPool, r->query_pool);
render_buffer_close(vk, &r->mesh.vbo);