xrt,st/oxr,u/space_overseer: Implement advanced xrLocateSpaces

v2:

xrt: Allow NULL space in locate_spaces

For action spaces, an xrt_space is found if
get_xrt_space -> get_xrt_space_action -> oxr_action_get_pose_input
finds an active input. If no input is active, the xrt_space will be NULL.

ipc: Check allocation of space_ids for locate_spaces

st/oxr: Fix memory leak in xrLocateSpaces with invalid spaces

v3: ipc: Return error when locate_spaces allocation fails

Part-of: <https://gitlab.freedesktop.org/monado/monado/-/merge_requests/2194>
This commit is contained in:
Christoph Haag 2024-04-09 01:00:40 +02:00 committed by Marge Bot
parent 0bd463c2af
commit 5ba58a5711
8 changed files with 496 additions and 37 deletions

View file

@ -23,6 +23,7 @@
#include "util/u_space_overseer.h"
#include <assert.h>
#include <math.h>
#include <pthread.h>
@ -534,6 +535,74 @@ locate_space(struct xrt_space_overseer *xso,
return XRT_SUCCESS;
}
static bool
pose_approx(const struct xrt_pose *a, const struct xrt_pose *b)
{
const float e = 0.00001f;
return fabsf(a->orientation.x - b->orientation.x) < e && //
fabsf(a->orientation.y - b->orientation.y) < e && //
fabsf(a->orientation.z - b->orientation.z) < e && //
fabsf(a->orientation.w - b->orientation.w) < e && //
fabsf(a->position.x - b->position.x) < e && //
fabsf(a->position.y - b->position.y) < e && //
fabsf(a->position.z - b->position.z) < e;
}
static int32_t
find_same_space_before(struct xrt_space **spaces, const struct xrt_pose *offsets, uint32_t space_index)
{
for (int32_t i = 0; i < (int32_t)space_index; i++) {
if (spaces[i] == spaces[space_index] && pose_approx(&offsets[i], &offsets[space_index])) {
return i;
}
}
return -1;
}
static xrt_result_t
locate_spaces(struct xrt_space_overseer *xso,
struct xrt_space *base_space,
const struct xrt_pose *base_offset,
uint64_t at_timestamp_ns,
struct xrt_space **spaces,
uint32_t space_count,
const struct xrt_pose *offsets,
struct xrt_space_relation *out_relations)
{
struct u_space_overseer *uso = u_space_overseer(xso);
struct u_space *ubase_space = u_space(base_space);
for (uint32_t i = 0; i < space_count; i++) {
// spaces are allowed to be NULL
if (spaces[i] == NULL) {
out_relations->relation_flags = XRT_SPACE_RELATION_BITMASK_NONE;
continue;
}
// crude optimization: If space ptr is equal to one already located, don't locate again, just copy
{
int32_t found = find_same_space_before(spaces, offsets, i);
if (found >= 0) {
out_relations[i] = out_relations[found];
continue;
}
}
struct u_space *uspace = u_space(spaces[i]);
struct xrt_relation_chain xrc = {0};
m_relation_chain_push_pose_if_not_identity(&xrc, &offsets[i]);
build_relation_chain(uso, &xrc, ubase_space, uspace, at_timestamp_ns);
m_relation_chain_push_inverted_pose_if_not_identity(&xrc, base_offset);
// For base_space =~= space (approx equals).
special_resolve(&xrc, &out_relations[i]);
}
return XRT_SUCCESS;
}
static xrt_result_t
locate_device(struct xrt_space_overseer *xso,
struct xrt_space *base_space,
@ -762,6 +831,7 @@ u_space_overseer_create(struct xrt_session_event_sink *broadcast)
uso->base.create_offset_space = create_offset_space;
uso->base.create_pose_space = create_pose_space;
uso->base.locate_space = locate_space;
uso->base.locate_spaces = locate_spaces;
uso->base.locate_device = locate_device;
uso->base.ref_space_inc = ref_space_inc;
uso->base.ref_space_dec = ref_space_dec;

View file

@ -159,6 +159,29 @@ struct xrt_space_overseer
const struct xrt_pose *offset,
struct xrt_space_relation *out_relation);
/*!
* Locate spaces in the base space.
*
* @see xrt_device::get_tracked_pose.
*
* @param[in] xso Owning space overseer.
* @param[in] base_space The space that we want the pose in.
* @param[in] base_offset Offset if any to the base space.
* @param[in] at_timestamp_ns At which time.
* @param[in] spaces The array of pointers to spaces to be located.
* @param[in] space_count The number of spaces to locate.
* @param[in] offsets Array of offset if any to the located spaces.
* @param[out] out_relations Array of resulting poses.
*/
xrt_result_t (*locate_spaces)(struct xrt_space_overseer *xso,
struct xrt_space *base_space,
const struct xrt_pose *base_offset,
uint64_t at_timestamp_ns,
struct xrt_space **spaces,
uint32_t space_count,
const struct xrt_pose *offsets,
struct xrt_space_relation *out_relations);
/*!
* Locate a the origin of the tracking space of a device, this is not
* the same as the device position. In other words, what is the position
@ -272,6 +295,27 @@ xrt_space_overseer_locate_space(struct xrt_space_overseer *xso,
return xso->locate_space(xso, base_space, base_offset, at_timestamp_ns, space, offset, out_relation);
}
/*!
* @copydoc xrt_space_overseer::locate_spaces
*
* Helper for calling through the function pointer.
*
* @public @memberof xrt_space_overseer
*/
static inline xrt_result_t
xrt_space_overseer_locate_spaces(struct xrt_space_overseer *xso,
struct xrt_space *base_space,
const struct xrt_pose *base_offset,
uint64_t at_timestamp_ns,
struct xrt_space **spaces,
uint32_t space_count,
const struct xrt_pose *offsets,
struct xrt_space_relation *out_relations)
{
return xso->locate_spaces(xso, base_space, base_offset, at_timestamp_ns, spaces, space_count, offsets,
out_relations);
}
/*!
* @copydoc xrt_space_overseer::locate_device
*

View file

@ -7,6 +7,9 @@
* @ingroup ipc_client
*/
#include "client/ipc_client.h"
#include "client/ipc_client_connection.h"
#include "shared/ipc_message_channel.h"
#include "xrt/xrt_defines.h"
#include "xrt/xrt_space.h"
@ -150,6 +153,67 @@ locate_space(struct xrt_space_overseer *xso,
IPC_CHK_ALWAYS_RET(icspo->ipc_c, xret, "ipc_call_space_locate_space");
}
static xrt_result_t
locate_spaces(struct xrt_space_overseer *xso,
struct xrt_space *base_space,
const struct xrt_pose *base_offset,
uint64_t at_timestamp_ns,
struct xrt_space **spaces,
uint32_t space_count,
const struct xrt_pose *offsets,
struct xrt_space_relation *out_relations)
{
struct ipc_client_space_overseer *icspo = ipc_client_space_overseer(xso);
struct ipc_connection *ipc_c = icspo->ipc_c;
xrt_result_t xret;
struct ipc_client_space *icsp_base_space = ipc_client_space(base_space);
uint32_t *space_ids = U_TYPED_ARRAY_CALLOC(uint32_t, space_count);
if (space_ids == NULL) {
IPC_ERROR(ipc_c, "Failed to allocate space_ids");
return XRT_ERROR_ALLOCATION;
}
ipc_client_connection_lock(ipc_c);
xret =
ipc_send_space_locate_spaces_locked(ipc_c, icsp_base_space->id, base_offset, space_count, at_timestamp_ns);
IPC_CHK_WITH_GOTO(ipc_c, xret, "ipc_send_space_locate_spaces_locked", locate_spaces_out);
enum xrt_result received_result = XRT_SUCCESS;
xret = ipc_receive(&ipc_c->imc, &received_result, sizeof(enum xrt_result));
IPC_CHK_WITH_GOTO(ipc_c, xret, "ipc_receive: Receive spaces allocation result", locate_spaces_out);
// now check if the service sent a success code or an error code about allocating memory for spaces.
xret = received_result;
IPC_CHK_WITH_GOTO(ipc_c, xret, "ipc_receive: service side spaces allocation failed", locate_spaces_out);
for (uint32_t i = 0; i < space_count; i++) {
if (spaces[i] == NULL) {
space_ids[i] = UINT32_MAX;
} else {
struct ipc_client_space *icsp_space = ipc_client_space(spaces[i]);
space_ids[i] = icsp_space->id;
}
}
xret = ipc_send(&ipc_c->imc, space_ids, sizeof(uint32_t) * space_count);
IPC_CHK_WITH_GOTO(ipc_c, xret, "ipc_send: Send spaces ids", locate_spaces_out);
xret = ipc_send(&ipc_c->imc, offsets, sizeof(struct xrt_pose) * space_count);
IPC_CHK_WITH_GOTO(ipc_c, xret, "ipc_send: Send spaces offsets", locate_spaces_out);
xret = ipc_receive(&ipc_c->imc, out_relations, sizeof(struct xrt_space_relation) * space_count);
IPC_CHK_WITH_GOTO(ipc_c, xret, "ipc_receive: Receive spaces relations", locate_spaces_out);
locate_spaces_out:
free(space_ids);
ipc_client_connection_unlock(ipc_c);
return xret;
}
static xrt_result_t
locate_device(struct xrt_space_overseer *xso,
struct xrt_space *base_space,
@ -265,6 +329,7 @@ ipc_client_space_overseer_create(struct ipc_connection *ipc_c)
icspo->base.create_offset_space = create_offset_space;
icspo->base.create_pose_space = create_pose_space;
icspo->base.locate_space = locate_space;
icspo->base.locate_spaces = locate_spaces;
icspo->base.locate_device = locate_device;
icspo->base.ref_space_inc = ref_space_inc;
icspo->base.ref_space_dec = ref_space_dec;

View file

@ -490,6 +490,108 @@ ipc_handle_space_locate_space(volatile struct ipc_client_state *ics,
out_relation); //
}
xrt_result_t
ipc_handle_space_locate_spaces(volatile struct ipc_client_state *ics,
uint32_t base_space_id,
const struct xrt_pose *base_offset,
uint32_t space_count,
uint64_t at_timestamp)
{
IPC_TRACE_MARKER();
struct ipc_message_channel *imc = (struct ipc_message_channel *)&ics->imc;
struct ipc_server *s = ics->server;
struct xrt_space_overseer *xso = ics->server->xso;
struct xrt_space *base_space = NULL;
struct xrt_space **xspaces = U_TYPED_ARRAY_CALLOC(struct xrt_space *, space_count);
struct xrt_pose *offsets = U_TYPED_ARRAY_CALLOC(struct xrt_pose, space_count);
struct xrt_space_relation *out_relations = U_TYPED_ARRAY_CALLOC(struct xrt_space_relation, space_count);
xrt_result_t xret;
os_mutex_lock(&ics->server->global_state.lock);
uint32_t *space_ids = U_TYPED_ARRAY_CALLOC(uint32_t, space_count);
// we need to send back whether allocation succeeded so the client knows whether to send more data
if (space_ids == NULL) {
xret = XRT_ERROR_ALLOCATION;
} else {
xret = XRT_SUCCESS;
}
xret = ipc_send(imc, &xret, sizeof(enum xrt_result));
if (xret != XRT_SUCCESS) {
IPC_ERROR(ics->server, "Failed to send spaces allocate result");
// Nothing else we can do
goto out_locate_spaces;
}
// only after sending the allocation result can we skip to the end in the allocation error case
if (space_ids == NULL) {
IPC_ERROR(s, "Failed to allocate space for receiving spaces ids");
goto out_locate_spaces;
}
xret = ipc_receive(imc, space_ids, space_count * sizeof(uint32_t));
if (xret != XRT_SUCCESS) {
IPC_ERROR(ics->server, "Failed to receive spaces ids");
// assume early abort is possible, i.e. client will not send more data for this request
goto out_locate_spaces;
}
xret = ipc_receive(imc, offsets, space_count * sizeof(struct xrt_pose));
if (xret != XRT_SUCCESS) {
IPC_ERROR(ics->server, "Failed to receive spaces offsets");
// assume early abort is possible, i.e. client will not send more data for this request
goto out_locate_spaces;
}
xret = validate_space_id(ics, base_space_id, &base_space);
if (xret != XRT_SUCCESS) {
U_LOG_E("Invalid base_space_id %d!", base_space_id);
// Client is receiving out_relations now, it will get xret on this receive.
goto out_locate_spaces;
}
for (uint32_t i = 0; i < space_count; i++) {
if (space_ids[i] == UINT32_MAX) {
xspaces[i] = NULL;
} else {
xret = validate_space_id(ics, space_ids[i], &xspaces[i]);
if (xret != XRT_SUCCESS) {
U_LOG_E("Invalid space_id space_ids[%d] = %d!", i, space_ids[i]);
// Client is receiving out_relations now, it will get xret on this receive.
goto out_locate_spaces;
}
}
}
xret = xrt_space_overseer_locate_spaces( //
xso, //
base_space, //
base_offset, //
at_timestamp, //
xspaces, //
space_count, //
offsets, //
out_relations); //
xret = ipc_send(imc, out_relations, sizeof(struct xrt_space_relation) * space_count);
if (xret != XRT_SUCCESS) {
IPC_ERROR(ics->server, "Failed to send spaces relations");
// Nothing else we can do
goto out_locate_spaces;
}
out_locate_spaces:
free(xspaces);
free(offsets);
free(out_relations);
os_mutex_unlock(&ics->server->global_state.lock);
return xret;
}
xrt_result_t
ipc_handle_space_locate_device(volatile struct ipc_client_state *ics,
uint32_t base_space_id,

View file

@ -131,6 +131,16 @@
]
},
"space_locate_spaces": {
"varlen": true,
"in": [
{"name": "base_space_id", "type": "uint32_t"},
{"name": "base_offset", "type": "struct xrt_pose"},
{"name": "space_count", "type": "uint32_t"},
{"name": "at_timestamp", "type": "uint64_t"}
]
},
"space_locate_device": {
"in": [
{"name": "base_space_id", "type": "uint32_t"},

View file

@ -249,12 +249,21 @@ oxr_xrDestroySpace(XrSpace space)
}
static XrResult
locate_spaces(XrSession session, const XrSpacesLocateInfo *locateInfo, XrSpaceLocations *spaceLocations)
verify_space(struct oxr_logger *log, XrSpace space, struct oxr_space **out_space)
{
struct oxr_space *spc;
OXR_VERIFY_SPACE_NOT_NULL(log, space, spc);
*out_space = spc;
return XR_SUCCESS;
}
static XrResult
locate_spaces(XrSession session, const XrSpacesLocateInfo *locateInfo, XrSpaceLocations *spaceLocations, char *fn)
{
struct oxr_space *spc;
struct oxr_space *baseSpc;
struct oxr_logger log;
OXR_VERIFY_SPACE_AND_INIT_LOG(&log, locateInfo->baseSpace, spc, "xrLocateSpacesKHR");
OXR_VERIFY_SPACE_AND_INIT_LOG(&log, locateInfo->baseSpace, spc, fn);
OXR_VERIFY_SESSION_NOT_LOST(&log, spc->sess);
OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, locateInfo, XR_TYPE_SPACES_LOCATE_INFO_KHR);
OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, spaceLocations, XR_TYPE_SPACE_LOCATIONS_KHR);
@ -286,43 +295,24 @@ locate_spaces(XrSession session, const XrSpacesLocateInfo *locateInfo, XrSpaceLo
}
for (uint32_t i = 0; i < locateInfo->spaceCount; i++) {
struct oxr_space *s;
OXR_VERIFY_SPACE_NOT_NULL(&log, locateInfo->spaces[i], s);
uint32_t space_count = locateInfo->spaceCount;
struct oxr_space **spaces = U_TYPED_ARRAY_CALLOC(struct oxr_space *, space_count);
XrSpaceVelocity v = {
.type = XR_TYPE_SPACE_VELOCITY,
.next = NULL,
};
void *next = NULL;
if (velocities) {
next = &v;
}
XrSpaceLocation l = {
.type = XR_TYPE_SPACE_LOCATION,
.next = next,
};
XrResult result = oxr_space_locate(&log, s, baseSpc, locateInfo->time, &l);
if (result == XR_SUCCESS) {
spaceLocations->locations[i].locationFlags = l.locationFlags;
spaceLocations->locations[i].pose = l.pose;
if (velocities) {
velocities->velocities[i].angularVelocity = v.angularVelocity;
velocities->velocities[i].linearVelocity = v.linearVelocity;
velocities->velocities[i].velocityFlags = v.velocityFlags;
}
} else {
return result;
XrResult res;
for (uint32_t i = 0; i < space_count; i++) {
res = verify_space(&log, locateInfo->spaces[i], &spaces[i]);
if (res != XR_SUCCESS) {
break;
}
}
return XR_SUCCESS;
if (res == XR_SUCCESS) {
res = oxr_spaces_locate(&log, spaces, space_count, baseSpc, locateInfo->time, spaceLocations);
}
free(spaces);
return res;
}
#ifdef OXR_HAVE_KHR_locate_spaces
@ -331,7 +321,7 @@ oxr_xrLocateSpacesKHR(XrSession session, const XrSpacesLocateInfoKHR *locateInfo
{
OXR_TRACE_MARKER();
return locate_spaces(session, locateInfo, spaceLocations);
return locate_spaces(session, locateInfo, spaceLocations, "xrLocateSpacesKHR");
}
#endif
@ -340,5 +330,5 @@ oxr_xrLocateSpaces(XrSession session, const XrSpacesLocateInfo *locateInfo, XrSp
{
OXR_TRACE_MARKER();
return locate_spaces(session, locateInfo, spaceLocations);
return locate_spaces(session, locateInfo, spaceLocations, "xrLocateSpaces");
}

View file

@ -891,6 +891,14 @@ XrResult
oxr_space_locate(
struct oxr_logger *log, struct oxr_space *spc, struct oxr_space *baseSpc, XrTime time, XrSpaceLocation *location);
XrResult
oxr_spaces_locate(struct oxr_logger *log,
struct oxr_space **spcs,
uint32_t spc_count,
struct oxr_space *baseSpc,
XrTime time,
XrSpaceLocations *locations);
/*!
* Locate the @ref xrt_device in the given base space, useful for implementing
* hand tracking location look ups and the like.

View file

@ -253,6 +253,176 @@ oxr_space_xdev_pose_create(struct oxr_logger *log,
*
*/
static void
free_spaces(struct xrt_space ***xtargets, struct xrt_pose **offsets, struct xrt_space_relation **results)
{
free(*xtargets);
*xtargets = NULL;
free(*offsets);
*offsets = NULL;
free(*results);
*results = NULL;
}
XrResult
oxr_spaces_locate(struct oxr_logger *log,
struct oxr_space **spcs,
uint32_t spc_count,
struct oxr_space *baseSpc,
XrTime time,
XrSpaceLocations *locations)
{
struct oxr_sink_logger slog = {0};
struct oxr_system *sys = baseSpc->sess->sys;
bool print = sys->inst->debug_spaces;
if (print) {
for (uint32_t i = 0; i < spc_count; i++) {
oxr_pp_space_indented(&slog, spcs[i], "space");
}
oxr_pp_space_indented(&slog, baseSpc, "baseSpace");
}
// Used in a lot of places.
XrSpaceVelocitiesKHR *vels =
OXR_GET_OUTPUT_FROM_CHAIN(locations->next, XR_TYPE_SPACE_VELOCITIES_KHR, XrSpaceVelocitiesKHR);
// XrEyeGazeSampleTimeEXT can not be chained anywhere in xrLocateSpaces
/*
* Seek knowledge about the spaces from the space overseer.
*/
struct xrt_space *xbase = NULL;
struct xrt_space **xspcs = U_TYPED_ARRAY_CALLOC(struct xrt_space *, spc_count);
struct xrt_pose *offsets = U_TYPED_ARRAY_CALLOC(struct xrt_pose, spc_count);
XrResult ret = XR_SUCCESS;
for (uint32_t i = 0; i < spc_count; i++) {
// Once we hit an error, we don't need to continue. Also make sure to not overwrite the error.
if (ret != XR_SUCCESS) {
break;
}
struct xrt_space *xt = NULL;
ret = get_xrt_space(log, spcs[i], &xt);
xspcs[i] = xt;
offsets[i] = spcs[i]->pose;
}
// Make sure not to overwrite error return
if (ret == XR_SUCCESS) {
ret = get_xrt_space(log, baseSpc, &xbase);
}
// Only fill this out if the above succeeded. Zero initialized means relation flags == 0.
struct xrt_space_relation *results = U_TYPED_ARRAY_CALLOC(struct xrt_space_relation, spc_count);
if (ret == XR_SUCCESS) {
// Convert at_time to monotonic and give to device.
uint64_t at_timestamp_ns = time_state_ts_to_monotonic_ns(sys->inst->timekeeping, time);
// Ask the space overseer to locate the spaces.
enum xrt_result xret = xrt_space_overseer_locate_spaces( //
sys->xso, //
xbase, //
&baseSpc->pose, //
at_timestamp_ns, //
xspcs, //
spc_count, //
offsets, //
results); //
if (xret != XRT_SUCCESS) {
//! @todo results locationFlags should still be 0. But should this hard fail? goto "Print"?
oxr_warn(log, "Failed to locate spaces (%d)", xret);
if (xret != XRT_SUCCESS) {
ret = XR_ERROR_RUNTIME_FAILURE;
if (xret == XRT_ERROR_ALLOCATION) {
//! @todo spec: function not specified to return out of memory
// ret = XR_ERROR_OUT_OF_MEMORY;
}
}
}
}
/*
* Validate results
*/
for (uint32_t i = 0; i < spc_count; i++) {
if (results[i].relation_flags == XRT_SPACE_RELATION_BITMASK_NONE) {
locations->locations[i].locationFlags = 0;
OXR_XRT_POSE_TO_XRPOSEF(XRT_POSE_IDENTITY, locations->locations[i].pose);
if (vels) {
vels->velocities[i].velocityFlags = 0;
U_ZERO(&vels->velocities[i].linearVelocity);
U_ZERO(&vels->velocities[i].angularVelocity);
}
oxr_slog(&slog, "\n\tReturning invalid pose locations->locations[%d]", i);
} else {
/*
* Combine and copy
*/
OXR_XRT_POSE_TO_XRPOSEF(results[i].pose, locations->locations[i].pose);
locations->locations[i].locationFlags =
xrt_to_xr_space_location_flags(results[i].relation_flags);
if (vels) {
vels->velocities[i].velocityFlags = 0;
if ((results[i].relation_flags & XRT_SPACE_RELATION_LINEAR_VELOCITY_VALID_BIT) != 0) {
vels->velocities[i].linearVelocity.x = results[i].linear_velocity.x;
vels->velocities[i].linearVelocity.y = results[i].linear_velocity.y;
vels->velocities[i].linearVelocity.z = results[i].linear_velocity.z;
vels->velocities[i].velocityFlags |= XR_SPACE_VELOCITY_LINEAR_VALID_BIT;
} else {
U_ZERO(&vels->velocities[i].linearVelocity);
}
if ((results[i].relation_flags & XRT_SPACE_RELATION_ANGULAR_VELOCITY_VALID_BIT) != 0) {
vels->velocities[i].angularVelocity.x = results[i].angular_velocity.x;
vels->velocities[i].angularVelocity.y = results[i].angular_velocity.y;
vels->velocities[i].angularVelocity.z = results[i].angular_velocity.z;
vels->velocities[i].velocityFlags |= XR_SPACE_VELOCITY_ANGULAR_VALID_BIT;
} else {
U_ZERO(&vels->velocities[i].angularVelocity);
}
}
oxr_pp_relation_indented(&slog, &results[i], "relation");
}
}
/*
* Print
*/
if (print) {
oxr_log_slog(log, &slog);
} else {
oxr_slog_cancel(&slog);
}
free_spaces(&xspcs, &offsets, &results);
if (ret != XR_SUCCESS) {
return ret;
}
// all spaces must be on the same session
return oxr_session_success_result(baseSpc->sess);
}
XrResult
oxr_space_locate(
struct oxr_logger *log, struct oxr_space *spc, struct oxr_space *baseSpc, XrTime time, XrSpaceLocation *location)