mirrors/monado
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/monado/monado.git synced 2025-01-29 18:08:29 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

xrt: Adds framework for face-tracking xrt-devices

Browse source
This commit is contained in:
Korcan Hussein 2024-03-06 10:33:30 +00:00
parent b622ba0979
commit 1a6c444a42
17 changed files with 133 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/xrt/auxiliary/util/u_pretty_print.c
View file

@ -1,4 +1,4 @@
// Copyright 2022-2023, Collabora, Ltd.
// Copyright 2022-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
@ -36,6 +36,7 @@ get_xrt_input_type_short_str(enum xrt_input_type type)
case XRT_INPUT_TYPE_BOOLEAN: return "BOOLEAN";
case XRT_INPUT_TYPE_POSE: return "POSE";
case XRT_INPUT_TYPE_HAND_TRACKING: return "HAND_TRACKING";
case XRT_INPUT_TYPE_FACE_TRACKING: return "FACE_TRACKING";
default: return "<UNKNOWN>";
}
}

16
src/xrt/include/xrt/xrt_defines.h
View file

@ -1,4 +1,4 @@
// Copyright 2019-2023, Collabora, Ltd.
// Copyright 2019-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
@ -768,7 +768,8 @@ enum xrt_device_type
XRT_DEVICE_TYPE_ANY_HAND_CONTROLLER,
XRT_DEVICE_TYPE_GENERIC_TRACKER,
XRT_DEVICE_TYPE_HAND_TRACKER,
XRT_DEVICE_TYPE_EYE_TRACKER
XRT_DEVICE_TYPE_EYE_TRACKER,
XRT_DEVICE_TYPE_FACE_TRACKER
};
/*!
@ -793,6 +794,8 @@ enum xrt_input_type
XRT_INPUT_TYPE_POSE = 0x05,
//! A tracked hand
XRT_INPUT_TYPE_HAND_TRACKING = 0x06,
//! A tracked face
XRT_INPUT_TYPE_FACE_TRACKING = 0x07
// clang-format on
};
@ -1114,6 +1117,8 @@ enum xrt_input_name
XRT_INPUT_OPPO_MR_THUMBSTICK_CLICK = XRT_INPUT_NAME(0x0510, BOOLEAN),
XRT_INPUT_OPPO_MR_THUMBSTICK_TOUCH = XRT_INPUT_NAME(0x0511, BOOLEAN),
XRT_INPUT_OPPO_MR_THUMBSTICK = XRT_INPUT_NAME(0x0512, VEC2_MINUS_ONE_TO_ONE),
XRT_INPUT_GENERIC_FACE_TRACKING = XRT_INPUT_NAME(0x0600, FACE_TRACKING),
// clang-format on
};
@ -1261,6 +1266,13 @@ enum xrt_output_type
#define XRT_OUTPUT_NAME(id, type) ((UINT32_C(id) << XRT_OUTPUT_TYPE_BITWIDTH) | (uint32_t)XRT_OUTPUT_TYPE_##type)
struct xrt_facial_expression_set
{
union {
struct xrt_facial_expression_empty_set* placeholder;
};
};
/*!
* Name of a output with a baked in type.
*

34
src/xrt/include/xrt/xrt_device.h
View file

@ -1,10 +1,11 @@
// Copyright 2019-2021, Collabora, Ltd.
// Copyright 2019-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
* @brief Header defining an xrt display or controller device.
* @author Jakob Bornecrantz <jakob@collabora.com>
* @author Moses Turner <mosesturner@protonmail.com>
* @author Korcan Hussein <korcan.hussein@collabora.com>
* @ingroup xrt_iface
*/
@ -265,7 +266,7 @@ struct xrt_device
bool ref_space_usage_supported;
bool form_factor_check_supported;
bool stage_supported;
bool face_tracking_supported;
/*
*
@ -338,6 +339,20 @@ struct xrt_device
struct xrt_hand_joint_set *out_value,
uint64_t *out_timestamp_ns);
/*!
* @brief Get the requested blend shape properties & weights for a face tracker
*
* @param[in] xdev The device.
* @param[in] facial_expression_type The facial expression data type (XR_FB_face_tracking,
* XR_HTC_facial_tracking, etc).
* @param[in] out_value Set of requested expression weights & blend shape properties.
*
* @see xrt_input_name
*/
xrt_result_t (*get_face_tracking)(struct xrt_device *xdev,
enum xrt_input_name facial_expression_type,
struct xrt_facial_expression_set *out_value);
/*!
* Set a output value.
*
@ -507,6 +522,21 @@ xrt_device_get_hand_tracking(struct xrt_device *xdev,
xdev->get_hand_tracking(xdev, name, desired_timestamp_ns, out_value, out_timestamp_ns);
}
/*!
* Helper function for @ref xrt_device::get_face_tracking.
*
* @copydoc xrt_device::get_face_tracking
*
* @public @memberof xrt_device
*/
static inline xrt_result_t
xrt_device_get_face_tracking(struct xrt_device *xdev,
enum xrt_input_name facial_expression_type,
struct xrt_facial_expression_set *out_value)
{
return xdev->get_face_tracking(xdev, facial_expression_type, out_value);
}
/*!
* Helper function for @ref xrt_device::set_output.
*

8
src/xrt/include/xrt/xrt_system.h
View file

@ -1,4 +1,4 @@
// Copyright 2020-2023, Collabora, Ltd.
// Copyright 2020-2024, Collabora, Ltd.
// Copyright 2023, NVIDIA CORPORATION.
// SPDX-License-Identifier: BSL-1.0
/*!
@ -252,6 +252,12 @@ struct xrt_system_devices
*/
struct xrt_device *eyes;
/*!
* An observing pointer to the device providing face tracking
* (optional).
*/
struct xrt_device *face;
/*!
* Devices providing optical (or otherwise more directly
* measured than from controller estimation) hand tracking.

20
src/xrt/ipc/client/ipc_client_device.c
View file

@ -1,9 +1,10 @@
// Copyright 2020-2023, Collabora, Ltd.
// Copyright 2020-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
* @brief IPC Client device.
* @author Jakob Bornecrantz <jakob@collabora.com>
* @author Korcan Hussein <korcan.hussein@collabora.com>
* @ingroup ipc_client
*/
@ -117,6 +118,21 @@ ipc_client_device_get_hand_tracking(struct xrt_device *xdev,
IPC_CHK_ONLY_PRINT(icd->ipc_c, xret, "ipc_call_device_get_hand_tracking");
}
static xrt_result_t
ipc_client_device_get_face_tracking(struct xrt_device *xdev,
enum xrt_input_name facial_expression_type,
struct xrt_facial_expression_set *out_value)
{
ipc_client_device_t *icd = ipc_client_device(xdev);
xrt_result_t xret = ipc_call_device_get_face_tracking( //
icd->ipc_c, //
icd->device_id, //
facial_expression_type, //
out_value); //
IPC_CHK_ALWAYS_RET(icd->ipc_c, xret, "ipc_call_device_get_face_tracking");
}
static void
ipc_client_device_get_view_poses(struct xrt_device *xdev,
const struct xrt_vec3 *default_eye_relation,
@ -166,6 +182,7 @@ ipc_client_device_create(struct ipc_connection *ipc_c, struct xrt_tracking_origi
icd->base.update_inputs = ipc_client_device_update_inputs;
icd->base.get_tracked_pose = ipc_client_device_get_tracked_pose;
icd->base.get_hand_tracking = ipc_client_device_get_hand_tracking;
icd->base.get_face_tracking = ipc_client_device_get_face_tracking;
icd->base.get_view_poses = ipc_client_device_get_view_poses;
icd->base.set_output = ipc_client_device_set_output;
icd->base.get_visibility_mask = ipc_client_device_get_visibility_mask;
@ -223,6 +240,7 @@ ipc_client_device_create(struct ipc_connection *ipc_c, struct xrt_tracking_origi
icd->base.position_tracking_supported = isdev->position_tracking_supported;
icd->base.hand_tracking_supported = isdev->hand_tracking_supported;
icd->base.eye_gaze_supported = isdev->eye_gaze_supported;
icd->base.face_tracking_supported = isdev->face_tracking_supported;
icd->base.force_feedback_supported = isdev->force_feedback_supported;
icd->base.stage_supported = isdev->stage_supported;

4
src/xrt/ipc/client/ipc_client_hmd.c
View file

@ -1,9 +1,10 @@
// Copyright 2020-2023, Collabora, Ltd.
// Copyright 2020-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
* @brief IPC Client HMD device.
* @author Jakob Bornecrantz <jakob@collabora.com>
* @author Korcan Hussein <korcan.hussein@collabora.com>
* @ingroup ipc_client
*/
@ -370,6 +371,7 @@ ipc_client_hmd_create(struct ipc_connection *ipc_c, struct xrt_tracking_origin *
ich->base.device_type = isdev->device_type;
ich->base.hand_tracking_supported = isdev->hand_tracking_supported;
ich->base.eye_gaze_supported = isdev->eye_gaze_supported;
ich->base.face_tracking_supported = isdev->face_tracking_supported;
ich->base.force_feedback_supported = isdev->force_feedback_supported;
ich->base.form_factor_check_supported = isdev->form_factor_check_supported;
ich->base.stage_supported = isdev->stage_supported;

3
src/xrt/ipc/client/ipc_client_instance.c
View file

@ -1,4 +1,4 @@
// Copyright 2020-2023, Collabora, Ltd.
// Copyright 2020-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
@ -168,6 +168,7 @@ ipc_client_instance_create_system(struct xrt_instance *xinst,
SET_ROLE(head);
SET_ROLE(eyes);
SET_ROLE(face);
SET_ROLE(hand_tracking.left);
SET_ROLE(hand_tracking.right);

14
src/xrt/ipc/server/ipc_server_handler.c
View file

@ -1,4 +1,4 @@
// Copyright 2020-2023, Collabora, Ltd.
// Copyright 2020-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
@ -1860,3 +1860,15 @@ ipc_handle_system_devices_get_roles(volatile struct ipc_client_state *ics, struc
{
return xrt_system_devices_get_roles(ics->server->xsysd, out_roles);
}
xrt_result_t
ipc_handle_device_get_face_tracking(volatile struct ipc_client_state *ics,
uint32_t id,
enum xrt_input_name facial_expression_type,
struct xrt_facial_expression_set *out_value)
{
const uint32_t device_id = id;
struct xrt_device *xdev = get_xdev(ics, device_id);
// Get facial expression data.
return xrt_device_get_face_tracking(xdev, facial_expression_type, out_value);
}

5
src/xrt/ipc/server/ipc_server_process.c
View file

@ -1,4 +1,4 @@
// Copyright 2020-2023, Collabora, Ltd.
// Copyright 2020-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
@ -6,6 +6,7 @@
* @author Pete Black <pblack@collabora.com>
* @author Jakob Bornecrantz <jakob@collabora.com>
* @author Rylie Pavlik <rylie.pavlik@collabora.com>
* @author Korcan Hussein <korcan.hussein@collabora.com>
* @ingroup ipc_server
*/
@ -352,6 +353,7 @@ init_shm(struct ipc_server *s)
isdev->force_feedback_supported = xdev->force_feedback_supported;
isdev->form_factor_check_supported = xdev->form_factor_check_supported;
isdev->eye_gaze_supported = xdev->eye_gaze_supported;
isdev->face_tracking_supported = xdev->face_tracking_supported;
isdev->stage_supported = xdev->stage_supported;
// Is this a HMD?
@ -429,6 +431,7 @@ init_shm(struct ipc_server *s)
// Assign all of the roles.
ism->roles.head = find_xdev_index(s, s->xsysd->static_roles.head);
ism->roles.eyes = find_xdev_index(s, s->xsysd->static_roles.eyes);
ism->roles.face = find_xdev_index(s, s->xsysd->static_roles.face);
ism->roles.hand_tracking.left = find_xdev_index(s, s->xsysd->static_roles.hand_tracking.left);
ism->roles.hand_tracking.right = find_xdev_index(s, s->xsysd->static_roles.hand_tracking.right);

5
src/xrt/ipc/shared/ipc_protocol.h
View file

@ -1,10 +1,11 @@
// Copyright 2020, Collabora, Ltd.
// Copyright 2020-2024 Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
* @brief Common protocol definition.
* @author Pete Black <pblack@collabora.com>
* @author Jakob Bornecrantz <jakob@collabora.com>
* @author Korcan Hussein <korcan.hussein@collabora.com>
* @ingroup ipc_shared
*/
@ -131,6 +132,7 @@ struct ipc_shared_device
bool position_tracking_supported;
bool hand_tracking_supported;
bool eye_gaze_supported;
bool face_tracking_supported;
bool force_feedback_supported;
bool form_factor_check_supported;
bool stage_supported;
@ -227,6 +229,7 @@ struct ipc_shared_memory
{
int32_t head;
int32_t eyes;
int32_t face;
struct
{

11
src/xrt/ipc/shared/proto.json
View file

@ -429,5 +429,16 @@
"out": [
{"name": "available", "type": "bool"}
]
},
"device_get_face_tracking": {
"in": [
{"name": "id", "type": "uint32_t"},
{"name": "facial_expression_type", "type": "enum xrt_input_name"}
],
"out": [
{"name": "value", "type": "struct xrt_facial_expression_set"}
]
}
}

4
src/xrt/state_trackers/oxr/oxr_defines.h
View file

@ -1,9 +1,10 @@
// Copyright 2018-2023, Collabora, Ltd.
// Copyright 2018-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
* @brief Shared internal defines and enums in the state tracker.
* @author Jakob Bornecrantz <jakob@collabora.com>
* @author Korcan Hussein <korcan.hussein@collabora.com>
* @ingroup oxr_main
*/
@ -27,6 +28,7 @@
#define OXR_XR_DEBUG_HTRACKER (*(uint64_t *)"oxrhtra\0")
#define OXR_XR_DEBUG_PASSTHROUGH (*(uint64_t *)"oxrpass\0")
#define OXR_XR_DEBUG_PASSTHROUGH_LAYER (*(uint64_t *)"oxrptla\0")
#define OXR_XR_DEBUG_FTRACKER (*(uint64_t *)"oxrftra\0")
// clang-format on
/*!

3
src/xrt/state_trackers/oxr/oxr_input.c
View file

@ -1080,6 +1080,9 @@ oxr_input_combine_input(struct oxr_session *sess,
case XRT_INPUT_TYPE_HAND_TRACKING:
// shouldn't be possible to get here
break;
case XRT_INPUT_TYPE_FACE_TRACKING:
// shouldn't be possible to get here
break;
}
}

8
src/xrt/state_trackers/oxr/oxr_objects.h
View file

@ -1,4 +1,4 @@
// Copyright 2018-2023, Collabora, Ltd.
// Copyright 2018-2024, Collabora, Ltd.
// Copyright 2023, NVIDIA CORPORATION.
// SPDX-License-Identifier: BSL-1.0
/*!
@ -1385,6 +1385,7 @@ struct oxr_system
// clang-format off
static inline struct xrt_device *get_role_head(struct oxr_system *sys) {return sys->xsysd->static_roles.head; }
static inline struct xrt_device *get_role_eyes(struct oxr_system *sys) {return sys->xsysd->static_roles.eyes; }
static inline struct xrt_device *get_role_face(struct oxr_system* sys) { return sys->xsysd->static_roles.face; }
static inline struct xrt_device *get_role_hand_tracking_left(struct oxr_system* sys) { return sys->xsysd->static_roles.hand_tracking.left; }
static inline struct xrt_device *get_role_hand_tracking_right(struct oxr_system* sys) { return sys->xsysd->static_roles.hand_tracking.right; }
// clang-format on
@ -1421,6 +1422,11 @@ get_role_profile_eyes(struct oxr_system *sys)
return XRT_DEVICE_INVALID;
}
static inline enum xrt_device_name
get_role_profile_face(struct oxr_system *sys)
{
return XRT_DEVICE_INVALID;
}
static inline enum xrt_device_name
get_role_profile_hand_tracking_left(struct oxr_system *sys)
{
return XRT_DEVICE_INVALID;

4
src/xrt/state_trackers/prober/p_prober.c
View file

@ -1,9 +1,10 @@
// Copyright 2019-2022, Collabora, Ltd.
// Copyright 2019-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
* @brief Main prober code.
* @author Jakob Bornecrantz <jakob@collabora.com>
* @author Korcan Hussein <korcan.hussein@collabora.com>
* @ingroup st_prober
*/
@ -856,6 +857,7 @@ print_system_devices(u_pp_delegate_t dg, struct xrt_system_devices *xsysd)
P(head);
P(eyes);
P(face);
PD(left);
PD(right);
PD(gamepad);

3
src/xrt/state_trackers/steamvr_drv/ovrd_driver.cpp
View file

@ -1,4 +1,4 @@
// Copyright 2020,2023 Collabora, Ltd.
// Copyright 2020,2024 Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
@ -681,6 +681,7 @@ public:
case XRT_INPUT_TYPE_POSE:
//! @todo how to handle poses?
case XRT_INPUT_TYPE_HAND_TRACKING:
case XRT_INPUT_TYPE_FACE_TRACKING:
case XRT_INPUT_TYPE_VEC3_MINUS_ONE_TO_ONE: break;
}
}

5
src/xrt/targets/cli/cli_cmd_test.c
View file

@ -1,9 +1,10 @@
// Copyright 2019-2023, Collabora, Ltd.
// Copyright 2019-2024, Collabora, Ltd.
// SPDX-License-Identifier: BSL-1.0
/*!
* @file
* @brief Prints a list of found devices and tests opening some of them.
* @author Jakob Bornecrantz <jakob@collabora.com>
* @author Korcan Hussein <korcan.hussein@collabora.com>
*/
#include "xrt/xrt_space.h"
@ -131,6 +132,8 @@ cli_cmd_test(int argc, const char **argv)
} while (false)
PRINT_ROLE(head, " ");
PRINT_ROLE(eyes, " ");
PRINT_ROLE(face, " ");
PRINT_DYNR(left, " ");
PRINT_DYNR(right, " ");
PRINT_DYNR(gamepad, " ");