mirrors/monado
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/monado/monado.git synced 2025-01-29 18:08:29 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

xrt: Adds framework for body-tracking xrt-devices

Browse source
This commit is contained in:
Korcan Hussein 2024-03-19 11:38:23 +00:00
parent ff2b30b7ee
commit 4d8cc7dffd
16 changed files with 196 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/xrt/auxiliary/util/u_pretty_print.c
View file

@ -37,6 +37,7 @@ get_xrt_input_type_short_str(enum xrt_input_type type)
case XRT_INPUT_TYPE_POSE: return "POSE";
case XRT_INPUT_TYPE_HAND_TRACKING: return "HAND_TRACKING";
case XRT_INPUT_TYPE_FACE_TRACKING: return "FACE_TRACKING";
case XRT_INPUT_TYPE_BODY_TRACKING: return "BODY_TRACKING";
default: return "<UNKNOWN>";
}
}

27
src/xrt/include/xrt/xrt_defines.h
View file

@ -772,7 +772,8 @@ enum xrt_device_type
XRT_DEVICE_TYPE_GENERIC_TRACKER,
XRT_DEVICE_TYPE_HAND_TRACKER,
XRT_DEVICE_TYPE_EYE_TRACKER,
XRT_DEVICE_TYPE_FACE_TRACKER
XRT_DEVICE_TYPE_FACE_TRACKER,
XRT_DEVICE_TYPE_BODY_TRACKER,
};
/*!
@ -798,7 +799,9 @@ enum xrt_input_type
//! A tracked hand
XRT_INPUT_TYPE_HAND_TRACKING = 0x06,
//! A tracked face
XRT_INPUT_TYPE_FACE_TRACKING = 0x07
XRT_INPUT_TYPE_FACE_TRACKING = 0x07,
//! A tracked body
XRT_INPUT_TYPE_BODY_TRACKING = 0x08,
// clang-format on
};
@ -1118,7 +1121,9 @@ enum xrt_input_type
_(XRT_INPUT_GENERIC_FACE_TRACKING , XRT_INPUT_NAME(0x0600, FACE_TRACKING)) \
\
_(XRT_INPUT_HTC_EYE_FACE_TRACKING , XRT_INPUT_NAME(0x0601, FACE_TRACKING)) \
_(XRT_INPUT_HTC_LIP_FACE_TRACKING , XRT_INPUT_NAME(0x0602, FACE_TRACKING))
_(XRT_INPUT_HTC_LIP_FACE_TRACKING , XRT_INPUT_NAME(0x0602, FACE_TRACKING)) \
\
_(XRT_INPUT_GENERIC_BODY_TRACKING , XRT_INPUT_NAME(0x0700, BODY_TRACKING))
// clang-format on
@ -1380,6 +1385,22 @@ struct xrt_facial_expression_set
};
};
// structure is a container to represent the body skeleton in T-pose including the joint hierarchy,
// can have info such as skeleton scale and proportions
struct xrt_body_skeleton
{
union {
struct xrt_body_empty_skeleton *placeholder;
};
};
struct xrt_body_joint_set
{
union {
struct xrt_body_joint_empty_set *placeholder;
};
};
/*!
* Name of a output with a baked in type.
*

69
src/xrt/include/xrt/xrt_device.h
View file

@ -268,6 +268,7 @@ struct xrt_device
bool form_factor_check_supported;
bool stage_supported;
bool face_tracking_supported;
bool body_tracking_supported;
/*
*
@ -354,6 +355,37 @@ struct xrt_device
enum xrt_input_name facial_expression_type,
struct xrt_facial_expression_set *out_value);
/*!
* @brief Get the body skeleton in T-pose, used to query the skeleton hierarchy, scale, proportions etc
*
* @param[in] xdev The device.
* @param[in] body_tracking_type The body joint set type (XR_FB_body_tracking,
* XR_META_body_tracking_full_body, etc).
* @param[in] out_value The body skeleton hierarchy/properties.
*
* @see xrt_input_name
*/
xrt_result_t (*get_body_skeleton)(struct xrt_device *xdev,
enum xrt_input_name body_tracking_type,
struct xrt_body_skeleton *out_value);
/*!
* @brief Get the joint locations for a body tracker
*
* @param[in] xdev The device.
* @param[in] body_tracking_type The body joint set type (XR_FB_body_tracking,
* XR_META_body_tracking_full_body, etc).
* @param[in] desired_timestamp_ns If the device can predict or has a history
* of locations, this is when the caller
* @param[in] out_value Set of body joint locations & properties.
*
* @see xrt_input_name
*/
xrt_result_t (*get_body_joints)(struct xrt_device *xdev,
enum xrt_input_name body_tracking_type,
uint64_t desired_timestamp_ns,
struct xrt_body_joint_set *out_value);
/*!
* Set a output value.
*
@ -539,6 +571,43 @@ xrt_device_get_face_tracking(struct xrt_device *xdev,
return xdev->get_face_tracking(xdev, facial_expression_type, out_value);
}
/*!
* Helper function for @ref xrt_device::get_body_skeleton.
*
* @copydoc xrt_device::get_body_skeleton
*
* @public @memberof xrt_device
*/
static inline xrt_result_t
xrt_device_get_body_skeleton(struct xrt_device *xdev,
enum xrt_input_name body_tracking_type,
struct xrt_body_skeleton *out_value)
{
if (xdev->get_body_skeleton == NULL) {
return XRT_ERROR_DEVICE_FUNCTION_NOT_IMPLEMENTED;
}
return xdev->get_body_skeleton(xdev, body_tracking_type, out_value);
}
/*!
* Helper function for @ref xrt_device::get_body_joints.
*
* @copydoc xrt_device::get_body_joints
*
* @public @memberof xrt_device
*/
static inline xrt_result_t
xrt_device_get_body_joints(struct xrt_device *xdev,
enum xrt_input_name body_tracking_type,
uint64_t desired_timestamp_ns,
struct xrt_body_joint_set *out_value)
{
if (xdev->get_body_joints == NULL) {
return XRT_ERROR_DEVICE_FUNCTION_NOT_IMPLEMENTED;
}
return xdev->get_body_joints(xdev, body_tracking_type, desired_timestamp_ns, out_value);
}
/*!
* Helper function for @ref xrt_device::set_output.
*

6
src/xrt/include/xrt/xrt_system.h
View file

@ -258,6 +258,12 @@ struct xrt_system_devices
*/
struct xrt_device *face;
/*!
* An observing pointer to the device providing body tracking
* (optional).
*/
struct xrt_device *body;
/*!
* Devices providing optical (or otherwise more directly
* measured than from controller estimation) hand tracking.

35
src/xrt/ipc/client/ipc_client_device.c
View file

@ -133,6 +133,38 @@ ipc_client_device_get_face_tracking(struct xrt_device *xdev,
IPC_CHK_ALWAYS_RET(icd->ipc_c, xret, "ipc_call_device_get_face_tracking");
}
static xrt_result_t
ipc_client_device_get_body_skeleton(struct xrt_device *xdev,
enum xrt_input_name body_tracking_type,
struct xrt_body_skeleton *out_value)
{
ipc_client_device_t *icd = ipc_client_device(xdev);
xrt_result_t xret = ipc_call_device_get_body_skeleton( //
icd->ipc_c, //
icd->device_id, //
body_tracking_type, //
out_value); //
IPC_CHK_ALWAYS_RET(icd->ipc_c, xret, "ipc_call_device_get_body_skeleton");
}
static xrt_result_t
ipc_client_device_get_body_joints(struct xrt_device *xdev,
enum xrt_input_name body_tracking_type,
uint64_t desired_timestamp_ns,
struct xrt_body_joint_set *out_value)
{
ipc_client_device_t *icd = ipc_client_device(xdev);
xrt_result_t xret = ipc_call_device_get_body_joints( //
icd->ipc_c, //
icd->device_id, //
body_tracking_type, //
desired_timestamp_ns, //
out_value); //
IPC_CHK_ALWAYS_RET(icd->ipc_c, xret, "ipc_call_device_get_body_joints");
}
static void
ipc_client_device_get_view_poses(struct xrt_device *xdev,
const struct xrt_vec3 *default_eye_relation,
@ -183,6 +215,8 @@ ipc_client_device_create(struct ipc_connection *ipc_c, struct xrt_tracking_origi
icd->base.get_tracked_pose = ipc_client_device_get_tracked_pose;
icd->base.get_hand_tracking = ipc_client_device_get_hand_tracking;
icd->base.get_face_tracking = ipc_client_device_get_face_tracking;
icd->base.get_body_skeleton = ipc_client_device_get_body_skeleton;
icd->base.get_body_joints = ipc_client_device_get_body_joints;
icd->base.get_view_poses = ipc_client_device_get_view_poses;
icd->base.set_output = ipc_client_device_set_output;
icd->base.get_visibility_mask = ipc_client_device_get_visibility_mask;
@ -241,6 +275,7 @@ ipc_client_device_create(struct ipc_connection *ipc_c, struct xrt_tracking_origi
icd->base.hand_tracking_supported = isdev->hand_tracking_supported;
icd->base.eye_gaze_supported = isdev->eye_gaze_supported;
icd->base.face_tracking_supported = isdev->face_tracking_supported;
icd->base.body_tracking_supported = isdev->body_tracking_supported;
icd->base.force_feedback_supported = isdev->force_feedback_supported;
icd->base.stage_supported = isdev->stage_supported;

1
src/xrt/ipc/client/ipc_client_hmd.c
View file

@ -373,6 +373,7 @@ ipc_client_hmd_create(struct ipc_connection *ipc_c, struct xrt_tracking_origin *
ich->base.hand_tracking_supported = isdev->hand_tracking_supported;
ich->base.eye_gaze_supported = isdev->eye_gaze_supported;
ich->base.face_tracking_supported = isdev->face_tracking_supported;
ich->base.body_tracking_supported = isdev->body_tracking_supported;
ich->base.force_feedback_supported = isdev->force_feedback_supported;
ich->base.form_factor_check_supported = isdev->form_factor_check_supported;
ich->base.stage_supported = isdev->stage_supported;

1
src/xrt/ipc/client/ipc_client_instance.c
View file

@ -169,6 +169,7 @@ ipc_client_instance_create_system(struct xrt_instance *xinst,
SET_ROLE(head);
SET_ROLE(eyes);
SET_ROLE(face);
SET_ROLE(body);
SET_ROLE(hand_tracking.left);
SET_ROLE(hand_tracking.right);

21
src/xrt/ipc/server/ipc_server_handler.c
View file

@ -1888,3 +1888,24 @@ ipc_handle_device_get_face_tracking(volatile struct ipc_client_state *ics,
// Get facial expression data.
return xrt_device_get_face_tracking(xdev, facial_expression_type, out_value);
}
xrt_result_t
ipc_handle_device_get_body_skeleton(volatile struct ipc_client_state *ics,
uint32_t id,
enum xrt_input_name body_tracking_type,
struct xrt_body_skeleton *out_value)
{
struct xrt_device *xdev = get_xdev(ics, id);
return xrt_device_get_body_skeleton(xdev, body_tracking_type, out_value);
}
xrt_result_t
ipc_handle_device_get_body_joints(volatile struct ipc_client_state *ics,
uint32_t id,
enum xrt_input_name body_tracking_type,
uint64_t desired_timestamp_ns,
struct xrt_body_joint_set *out_value)
{
struct xrt_device *xdev = get_xdev(ics, id);
return xrt_device_get_body_joints(xdev, body_tracking_type, desired_timestamp_ns, out_value);
}

2
src/xrt/ipc/server/ipc_server_process.c
View file

@ -354,6 +354,7 @@ init_shm(struct ipc_server *s)
isdev->form_factor_check_supported = xdev->form_factor_check_supported;
isdev->eye_gaze_supported = xdev->eye_gaze_supported;
isdev->face_tracking_supported = xdev->face_tracking_supported;
isdev->body_tracking_supported = xdev->body_tracking_supported;
isdev->stage_supported = xdev->stage_supported;
// Is this a HMD?
@ -434,6 +435,7 @@ init_shm(struct ipc_server *s)
ism->roles.head = find_xdev_index(s, s->xsysd->static_roles.head);
ism->roles.eyes = find_xdev_index(s, s->xsysd->static_roles.eyes);
ism->roles.face = find_xdev_index(s, s->xsysd->static_roles.face);
ism->roles.body = find_xdev_index(s, s->xsysd->static_roles.body);
ism->roles.hand_tracking.left = find_xdev_index(s, s->xsysd->static_roles.hand_tracking.left);
ism->roles.hand_tracking.right = find_xdev_index(s, s->xsysd->static_roles.hand_tracking.right);

2
src/xrt/ipc/shared/ipc_protocol.h
View file

@ -133,6 +133,7 @@ struct ipc_shared_device
bool hand_tracking_supported;
bool eye_gaze_supported;
bool face_tracking_supported;
bool body_tracking_supported;
bool force_feedback_supported;
bool form_factor_check_supported;
bool stage_supported;
@ -230,6 +231,7 @@ struct ipc_shared_memory
int32_t head;
int32_t eyes;
int32_t face;
int32_t body;
struct
{

21
src/xrt/ipc/shared/proto.json
View file

@ -449,6 +449,27 @@
"out": [
{"name": "value", "type": "struct xrt_facial_expression_set"}
]
},
"device_get_body_skeleton": {
"in": [
{"name": "id", "type": "uint32_t"},
{"name": "body_tracking_type", "type": "enum xrt_input_name"}
],
"out": [
{"name": "value", "type": "struct xrt_body_skeleton"}
]
},
"device_get_body_joints": {
"in": [
{"name": "id", "type": "uint32_t"},
{"name": "body_tracking_type", "type": "enum xrt_input_name"},
{"name": "desired_timestamp_ns", "type": "uint64_t"}
],
"out": [
{"name": "value", "type": "struct xrt_body_joint_set"}
]
}
}

2
src/xrt/state_trackers/oxr/oxr_defines.h
View file

@ -29,6 +29,8 @@
#define OXR_XR_DEBUG_PASSTHROUGH (*(uint64_t *)"oxrpass\0")
#define OXR_XR_DEBUG_PASSTHROUGH_LAYER (*(uint64_t *)"oxrptla\0")
#define OXR_XR_DEBUG_FTRACKER (*(uint64_t *)"oxrftra\0")
// body tracker
#define OXR_XR_DEBUG_BTRACKER (*(uint64_t *)"oxrbtra\0")
// clang-format on
/*!

3
src/xrt/state_trackers/oxr/oxr_input.c
View file

@ -1083,6 +1083,9 @@ oxr_input_combine_input(struct oxr_session *sess,
case XRT_INPUT_TYPE_FACE_TRACKING:
// shouldn't be possible to get here
break;
case XRT_INPUT_TYPE_BODY_TRACKING:
// shouldn't be possible to get here
break;
}
}

6
src/xrt/state_trackers/oxr/oxr_objects.h
View file

@ -1415,6 +1415,7 @@ struct oxr_system
static inline struct xrt_device *get_role_head(struct oxr_system *sys) {return sys->xsysd->static_roles.head; }
static inline struct xrt_device *get_role_eyes(struct oxr_system *sys) {return sys->xsysd->static_roles.eyes; }
static inline struct xrt_device *get_role_face(struct oxr_system* sys) { return sys->xsysd->static_roles.face; }
static inline struct xrt_device *get_role_body(struct oxr_system* sys) { return sys->xsysd->static_roles.body; }
static inline struct xrt_device *get_role_hand_tracking_left(struct oxr_system* sys) { return sys->xsysd->static_roles.hand_tracking.left; }
static inline struct xrt_device *get_role_hand_tracking_right(struct oxr_system* sys) { return sys->xsysd->static_roles.hand_tracking.right; }
// clang-format on
@ -1456,6 +1457,11 @@ get_role_profile_face(struct oxr_system *sys)
return XRT_DEVICE_INVALID;
}
static inline enum xrt_device_name
get_role_profile_body(struct oxr_system *sys)
{
return XRT_DEVICE_INVALID;
}
static inline enum xrt_device_name
get_role_profile_hand_tracking_left(struct oxr_system *sys)
{
return XRT_DEVICE_INVALID;

1
src/xrt/state_trackers/steamvr_drv/ovrd_driver.cpp
View file

@ -682,6 +682,7 @@ public:
//! @todo how to handle poses?
case XRT_INPUT_TYPE_HAND_TRACKING:
case XRT_INPUT_TYPE_FACE_TRACKING:
case XRT_INPUT_TYPE_BODY_TRACKING:
case XRT_INPUT_TYPE_VEC3_MINUS_ONE_TO_ONE: break;
}
}

1
src/xrt/targets/cli/cli_cmd_test.c
View file

@ -134,6 +134,7 @@ cli_cmd_test(int argc, const char **argv)
PRINT_ROLE(head, " ");
PRINT_ROLE(eyes, " ");
PRINT_ROLE(face, " ");
PRINT_ROLE(body, " ");
PRINT_DYNR(left, " ");
PRINT_DYNR(right, " ");
PRINT_DYNR(gamepad, " ");