From 50f3816cf2ef02f521512f2978d62deeb24ac066 Mon Sep 17 00:00:00 2001 From: Korcan Hussein Date: Fri, 8 Mar 2024 11:48:40 +0000 Subject: [PATCH] xrt: Adds support for XR_HTC_facial_tracking xrt-devices --- CMakeLists.txt | 6 +- scripts/generate_oxr_ext_support.py | 1 + scripts/mapping.imp | 7 + src/xrt/compositor/multi/comp_multi_system.c | 3 +- src/xrt/include/xrt/xrt_compositor.h | 3 +- .../include/xrt/xrt_config_build.h.cmake_in | 4 +- src/xrt/include/xrt/xrt_defines.h | 100 ++++++++++- src/xrt/include/xrt/xrt_instance.h | 3 +- src/xrt/ipc/server/ipc_server_handler.c | 4 + src/xrt/state_trackers/oxr/oxr_api_funcs.h | 20 ++- .../state_trackers/oxr/oxr_api_negotiate.c | 9 +- src/xrt/state_trackers/oxr/oxr_api_session.c | 168 +++++++++++++++++- src/xrt/state_trackers/oxr/oxr_api_verify.h | 5 +- .../oxr/oxr_extension_support.h | 16 +- src/xrt/state_trackers/oxr/oxr_instance.c | 5 +- src/xrt/state_trackers/oxr/oxr_objects.h | 43 +++++ src/xrt/state_trackers/oxr/oxr_session.c | 5 +- src/xrt/state_trackers/oxr/oxr_system.c | 47 ++++- 18 files changed, 433 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac02dd341..d62409720 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2018-2023, Collabora, Ltd. +# Copyright 2018-2024, Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 cmake_minimum_required(VERSION 3.10.2) @@ -364,6 +364,9 @@ option( XRT_FEATURE_OPENXR_VULKAN_SWAPCHAIN_FORMAT_LIST "Enable support for the XR_KHR_vulkan_swapchain_format_list extension" ON ) +if(NOT DEFINED XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC) + set(XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC OFF) +endif() # Interaction extension support. if(NOT DEFINED XRT_FEATURE_OPENXR_INTERACTION_EXT_EYE_GAZE) @@ -618,6 +621,7 @@ message(STATUS "# FEATURE_DEBUG_GUI: ${XRT_FEATURE message(STATUS "# FEATURE_OPENXR: ${XRT_FEATURE_OPENXR}") message(STATUS "# FEATURE_OPENXR_DEBUG_UTILS: ${XRT_FEATURE_OPENXR_DEBUG_UTILS}") message(STATUS "# FEATURE_OPENXR_DISPLAY_REFRESH_RATE: ${XRT_FEATURE_OPENXR_DISPLAY_REFRESH_RATE}") +message(STATUS "# FEATURE_OPENXR_FACIAL_TRACKING_HTC: ${XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC}") message(STATUS "# FEATURE_OPENXR_FORCE_FEEDBACK_CURL: ${XRT_FEATURE_OPENXR_FORCE_FEEDBACK_CURL}") message(STATUS "# FEATURE_OPENXR_HEADLESS: ${XRT_FEATURE_OPENXR_HEADLESS}") message(STATUS "# FEATURE_OPENXR_INTERACTION_EXT_EYE_GAZE: ${XRT_FEATURE_OPENXR_INTERACTION_EXT_EYE_GAZE}") diff --git a/scripts/generate_oxr_ext_support.py b/scripts/generate_oxr_ext_support.py index 7c7b5920b..3fd1c4d80 100755 --- a/scripts/generate_oxr_ext_support.py +++ b/scripts/generate_oxr_ext_support.py @@ -84,6 +84,7 @@ EXTENSIONS = ( ['XR_MNDX_force_feedback_curl', 'XRT_FEATURE_OPENXR_FORCE_FEEDBACK_CURL'], ['XR_MNDX_hydra', 'XRT_FEATURE_OPENXR_INTERACTION_MNDX'], ['XR_MNDX_system_buttons', 'XRT_FEATURE_OPENXR_INTERACTION_MNDX'], + ['XR_HTC_facial_tracking', 'XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC'], ) diff --git a/scripts/mapping.imp b/scripts/mapping.imp index 941f96302..5dcdfd7be 100644 --- a/scripts/mapping.imp +++ b/scripts/mapping.imp @@ -34,6 +34,7 @@ { symbol: ["XRT_FEATURE_OPENXR_DEBUG_UTILS", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_DISPLAY_REFRESH_RATE", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_FORCE_FEEDBACK_CURL", "public", "\"xrt/xrt_config_build.h\"", "public"] }, +{ symbol: ["XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_HEADLESS", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_INTERACTION_EXT_EYE_GAZE", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_INTERACTION_EXT_HAND", "public", "\"xrt/xrt_config_build.h\"", "public"] }, @@ -43,15 +44,21 @@ { symbol: ["XRT_FEATURE_OPENXR_INTERACTION_MSFT_HAND", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_INTERACTION_OPPO", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_INTERACTION_WINMR", "public", "\"xrt/xrt_config_build.h\"", "public"] }, +{ symbol: ["XRT_FEATURE_OPENXR_LAYER_COLOR_SCALE_BIAS", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_LAYER_CUBE", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_LAYER_CYLINDER", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_LAYER_DEPTH", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_LAYER_EQUIRECT1", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_LAYER_EQUIRECT2", "public", "\"xrt/xrt_config_build.h\"", "public"] }, +{ symbol: ["XRT_FEATURE_OPENXR_LAYER_FB_ALPHA_BLEND", "public", "\"xrt/xrt_config_build.h\"", "public"] }, +{ symbol: ["XRT_FEATURE_OPENXR_LAYER_FB_IMAGE_LAYOUT", "public", "\"xrt/xrt_config_build.h\"", "public"] }, +{ symbol: ["XRT_FEATURE_OPENXR_LAYER_FB_SETTINGS", "public", "\"xrt/xrt_config_build.h\"", "public"] }, +{ symbol: ["XRT_FEATURE_OPENXR_LAYER_FB_DEPTH_TEST", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_OVERLAY", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_SPACE_LOCAL_FLOOR", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_SPACE_UNBOUNDED", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_OPENXR_VISIBILITY_MASK", "public", "\"xrt/xrt_config_build.h\"", "public"] }, +{ symbol: ["XRT_FEATURE_OPENXR_VULKAN_SWAPCHAIN_FORMAT_LIST", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_RENDERDOC", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_SERVICE", "public", "\"xrt/xrt_config_build.h\"", "public"] }, { symbol: ["XRT_FEATURE_SERVICE_SYSTEMD", "public", "\"xrt/xrt_config_build.h\"", "public"] }, diff --git a/src/xrt/compositor/multi/comp_multi_system.c b/src/xrt/compositor/multi/comp_multi_system.c index 93435a040..b6a7cc40d 100644 --- a/src/xrt/compositor/multi/comp_multi_system.c +++ b/src/xrt/compositor/multi/comp_multi_system.c @@ -1,4 +1,4 @@ -// Copyright 2019-2023, Collabora, Ltd. +// Copyright 2019-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -375,6 +375,7 @@ update_session_state_locked(struct multi_system_compositor *msc) .ext_hand_tracking_enabled = false, .ext_eye_gaze_interaction_enabled = false, .ext_hand_interaction_enabled = false, + .htc_facial_tracking_enabled = false, }; switch (msc->sessions.state) { diff --git a/src/xrt/include/xrt/xrt_compositor.h b/src/xrt/include/xrt/xrt_compositor.h index d6c03ba45..c85054391 100644 --- a/src/xrt/include/xrt/xrt_compositor.h +++ b/src/xrt/include/xrt/xrt_compositor.h @@ -1,4 +1,4 @@ -// Copyright 2019-2023, Collabora, Ltd. +// Copyright 2019-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -959,6 +959,7 @@ struct xrt_begin_session_info bool ext_hand_tracking_enabled; bool ext_eye_gaze_interaction_enabled; bool ext_hand_interaction_enabled; + bool htc_facial_tracking_enabled; }; /*! diff --git a/src/xrt/include/xrt/xrt_config_build.h.cmake_in b/src/xrt/include/xrt/xrt_config_build.h.cmake_in index ce3c8f81a..2d3d07c9c 100644 --- a/src/xrt/include/xrt/xrt_config_build.h.cmake_in +++ b/src/xrt/include/xrt/xrt_config_build.h.cmake_in @@ -1,9 +1,10 @@ -// Copyright 2020-2023, Collabora, Ltd. +// Copyright 2020-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief What internal components we are building. * @author Jakob Bornecrantz + * @author Korcan Hussein * @ingroup xrt_iface */ @@ -31,6 +32,7 @@ #cmakedefine XRT_FEATURE_OPENXR_DEBUG_UTILS #cmakedefine XRT_FEATURE_OPENXR_DISPLAY_REFRESH_RATE #cmakedefine XRT_FEATURE_OPENXR_FORCE_FEEDBACK_CURL +#cmakedefine XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC #cmakedefine XRT_FEATURE_OPENXR_HEADLESS #cmakedefine XRT_FEATURE_OPENXR_INTERACTION_EXT_EYE_GAZE #cmakedefine XRT_FEATURE_OPENXR_INTERACTION_EXT_HAND diff --git a/src/xrt/include/xrt/xrt_defines.h b/src/xrt/include/xrt/xrt_defines.h index d39f2efe3..a207653c9 100644 --- a/src/xrt/include/xrt/xrt_defines.h +++ b/src/xrt/include/xrt/xrt_defines.h @@ -751,7 +751,10 @@ enum xrt_device_name XRT_DEVICE_DEPTHAI, //! XR_EXT_hand_interaction - XRT_DEVICE_EXT_HAND_INTERACTION + XRT_DEVICE_EXT_HAND_INTERACTION, + + //! XR_HTC_facial_tracking + XRT_DEVICE_HTC_FACE_TRACKING }; /*! @@ -1119,6 +1122,9 @@ enum xrt_input_name XRT_INPUT_OPPO_MR_THUMBSTICK = XRT_INPUT_NAME(0x0512, VEC2_MINUS_ONE_TO_ONE), XRT_INPUT_GENERIC_FACE_TRACKING = XRT_INPUT_NAME(0x0600, FACE_TRACKING), + + XRT_INPUT_HTC_EYE_FACE_TRACKING = XRT_INPUT_NAME(0x0601, FACE_TRACKING), + XRT_INPUT_HTC_LIP_FACE_TRACKING = XRT_INPUT_NAME(0x0602, FACE_TRACKING), // clang-format on }; @@ -1266,10 +1272,100 @@ enum xrt_output_type #define XRT_OUTPUT_NAME(id, type) ((UINT32_C(id) << XRT_OUTPUT_TYPE_BITWIDTH) | (uint32_t)XRT_OUTPUT_TYPE_##type) +enum xrt_eye_expression_htc +{ + XRT_EYE_EXPRESSION_LEFT_BLINK_HTC = 0, + XRT_EYE_EXPRESSION_LEFT_WIDE_HTC = 1, + XRT_EYE_EXPRESSION_RIGHT_BLINK_HTC = 2, + XRT_EYE_EXPRESSION_RIGHT_WIDE_HTC = 3, + XRT_EYE_EXPRESSION_LEFT_SQUEEZE_HTC = 4, + XRT_EYE_EXPRESSION_RIGHT_SQUEEZE_HTC = 5, + XRT_EYE_EXPRESSION_LEFT_DOWN_HTC = 6, + XRT_EYE_EXPRESSION_RIGHT_DOWN_HTC = 7, + XRT_EYE_EXPRESSION_LEFT_OUT_HTC = 8, + XRT_EYE_EXPRESSION_RIGHT_IN_HTC = 9, + XRT_EYE_EXPRESSION_LEFT_IN_HTC = 10, + XRT_EYE_EXPRESSION_RIGHT_OUT_HTC = 11, + XRT_EYE_EXPRESSION_LEFT_UP_HTC = 12, + XRT_EYE_EXPRESSION_RIGHT_UP_HTC = 13 +}; + +enum xrt_lip_expression_htc +{ + XRT_LIP_EXPRESSION_JAW_RIGHT_HTC = 0, + XRT_LIP_EXPRESSION_JAW_LEFT_HTC = 1, + XRT_LIP_EXPRESSION_JAW_FORWARD_HTC = 2, + XRT_LIP_EXPRESSION_JAW_OPEN_HTC = 3, + XRT_LIP_EXPRESSION_MOUTH_APE_SHAPE_HTC = 4, + XRT_LIP_EXPRESSION_MOUTH_UPPER_RIGHT_HTC = 5, + XRT_LIP_EXPRESSION_MOUTH_UPPER_LEFT_HTC = 6, + XRT_LIP_EXPRESSION_MOUTH_LOWER_RIGHT_HTC = 7, + XRT_LIP_EXPRESSION_MOUTH_LOWER_LEFT_HTC = 8, + XRT_LIP_EXPRESSION_MOUTH_UPPER_OVERTURN_HTC = 9, + XRT_LIP_EXPRESSION_MOUTH_LOWER_OVERTURN_HTC = 10, + XRT_LIP_EXPRESSION_MOUTH_POUT_HTC = 11, + XRT_LIP_EXPRESSION_MOUTH_SMILE_RIGHT_HTC = 12, + XRT_LIP_EXPRESSION_MOUTH_SMILE_LEFT_HTC = 13, + XRT_LIP_EXPRESSION_MOUTH_SAD_RIGHT_HTC = 14, + XRT_LIP_EXPRESSION_MOUTH_SAD_LEFT_HTC = 15, + XRT_LIP_EXPRESSION_CHEEK_PUFF_RIGHT_HTC = 16, + XRT_LIP_EXPRESSION_CHEEK_PUFF_LEFT_HTC = 17, + XRT_LIP_EXPRESSION_CHEEK_SUCK_HTC = 18, + XRT_LIP_EXPRESSION_MOUTH_UPPER_UPRIGHT_HTC = 19, + XRT_LIP_EXPRESSION_MOUTH_UPPER_UPLEFT_HTC = 20, + XRT_LIP_EXPRESSION_MOUTH_LOWER_DOWNRIGHT_HTC = 21, + XRT_LIP_EXPRESSION_MOUTH_LOWER_DOWNLEFT_HTC = 22, + XRT_LIP_EXPRESSION_MOUTH_UPPER_INSIDE_HTC = 23, + XRT_LIP_EXPRESSION_MOUTH_LOWER_INSIDE_HTC = 24, + XRT_LIP_EXPRESSION_MOUTH_LOWER_OVERLAY_HTC = 25, + XRT_LIP_EXPRESSION_TONGUE_LONGSTEP1_HTC = 26, + XRT_LIP_EXPRESSION_TONGUE_LEFT_HTC = 27, + XRT_LIP_EXPRESSION_TONGUE_RIGHT_HTC = 28, + XRT_LIP_EXPRESSION_TONGUE_UP_HTC = 29, + XRT_LIP_EXPRESSION_TONGUE_DOWN_HTC = 30, + XRT_LIP_EXPRESSION_TONGUE_ROLL_HTC = 31, + XRT_LIP_EXPRESSION_TONGUE_LONGSTEP2_HTC = 32, + XRT_LIP_EXPRESSION_TONGUE_UPRIGHT_MORPH_HTC = 33, + XRT_LIP_EXPRESSION_TONGUE_UPLEFT_MORPH_HTC = 34, + XRT_LIP_EXPRESSION_TONGUE_DOWNRIGHT_MORPH_HTC = 35, + XRT_LIP_EXPRESSION_TONGUE_DOWNLEFT_MORPH_HTC = 36 +}; + +enum xrt_facial_tracking_type_htc +{ + XRT_FACIAL_TRACKING_TYPE_EYE_DEFAULT_HTC = 1, + XRT_FACIAL_TRACKING_TYPE_LIP_DEFAULT_HTC = 2 +}; + +#define XRT_FACIAL_EXPRESSION_EYE_COUNT_HTC 14 +#define XRT_FACIAL_EXPRESSION_LIP_COUNT_HTC 37 + +struct xrt_facial_base_expression_set_htc +{ + uint64_t sample_time_ns; + bool is_active; +}; + +struct xrt_facial_eye_expression_set_htc +{ + struct xrt_facial_base_expression_set_htc base; + // ordered by xrt_eye_expression_htc + float expression_weights[XRT_FACIAL_EXPRESSION_EYE_COUNT_HTC]; +}; + +struct xrt_facial_lip_expression_set_htc +{ + struct xrt_facial_base_expression_set_htc base; + // ordered by xrt_lip_expression_htc + float expression_weights[XRT_FACIAL_EXPRESSION_LIP_COUNT_HTC]; +}; + struct xrt_facial_expression_set { union { - struct xrt_facial_expression_empty_set* placeholder; + struct xrt_facial_base_expression_set_htc base_expression_set_htc; + struct xrt_facial_eye_expression_set_htc eye_expression_set_htc; + struct xrt_facial_lip_expression_set_htc lip_expression_set_htc; }; }; diff --git a/src/xrt/include/xrt/xrt_instance.h b/src/xrt/include/xrt/xrt_instance.h index 5286f7dee..65f490a39 100644 --- a/src/xrt/include/xrt/xrt_instance.h +++ b/src/xrt/include/xrt/xrt_instance.h @@ -1,4 +1,4 @@ -// Copyright 2020-2023, Collabora, Ltd. +// Copyright 2020-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -43,6 +43,7 @@ struct xrt_instance_info bool ext_hand_tracking_enabled; bool ext_eye_gaze_interaction_enabled; bool ext_hand_interaction_enabled; + bool htc_facial_tracking_enabled; }; /*! diff --git a/src/xrt/ipc/server/ipc_server_handler.c b/src/xrt/ipc/server/ipc_server_handler.c index 3db3d04a6..2af19263a 100644 --- a/src/xrt/ipc/server/ipc_server_handler.c +++ b/src/xrt/ipc/server/ipc_server_handler.c @@ -200,6 +200,9 @@ ipc_handle_instance_describe_client(volatile struct ipc_client_state *ics, EXT(ext_hand_tracking_enabled); EXT(ext_eye_gaze_interaction_enabled); EXT(ext_hand_interaction_enabled); +#ifdef OXR_HAVE_HTC_facial_tracking + EXT(htc_facial_tracking_enabled); +#endif #undef EXT #undef PTT @@ -291,6 +294,7 @@ ipc_handle_session_begin(volatile struct ipc_client_state *ics) .ext_hand_tracking_enabled = ics->client_state.info.ext_hand_tracking_enabled, .ext_eye_gaze_interaction_enabled = ics->client_state.info.ext_eye_gaze_interaction_enabled, .ext_hand_interaction_enabled = ics->client_state.info.ext_hand_interaction_enabled, + .htc_facial_tracking_enabled = ics->client_state.info.htc_facial_tracking_enabled, }; return xrt_comp_begin_session(ics->xc, &begin_session_info); diff --git a/src/xrt/state_trackers/oxr/oxr_api_funcs.h b/src/xrt/state_trackers/oxr/oxr_api_funcs.h index d9f136e4b..cc86e6d0e 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_funcs.h +++ b/src/xrt/state_trackers/oxr/oxr_api_funcs.h @@ -1,9 +1,10 @@ -// Copyright 2018-2019, Collabora, Ltd. +// Copyright 2018-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Header defining all API functions. * @author Jakob Bornecrantz + * @author Korcan Hussein * @ingroup oxr_api */ @@ -608,6 +609,23 @@ oxr_xrPassthroughPauseFB(XrPassthroughFB passthrough); XRAPI_ATTR XrResult XRAPI_CALL oxr_xrPassthroughStartFB(XrPassthroughFB passthrough); #endif + +#ifdef OXR_HAVE_HTC_facial_tracking +//! OpenXR API function @ep{xrCreateFacialTrackerHTC} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrCreateFacialTrackerHTC(XrSession session, + const XrFacialTrackerCreateInfoHTC *createInfo, + XrFacialTrackerHTC *facialTracker); + +//! OpenXR API function @ep{xrDestroyFacialTrackerHTC} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrDestroyFacialTrackerHTC(XrFacialTrackerHTC facialTracker); + +//! OpenXR API function @ep{xrGetFacialExpressionsHTC} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrGetFacialExpressionsHTC(XrFacialTrackerHTC facialTracker, XrFacialExpressionsHTC *facialExpressions); +#endif + /*! * @} */ diff --git a/src/xrt/state_trackers/oxr/oxr_api_negotiate.c b/src/xrt/state_trackers/oxr/oxr_api_negotiate.c index c9fdff82c..2b3b04974 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_negotiate.c +++ b/src/xrt/state_trackers/oxr/oxr_api_negotiate.c @@ -1,10 +1,11 @@ -// Copyright 2018-2019, Collabora, Ltd. +// Copyright 2018-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief File for negotiating with the loader. * @author Rylie Pavlik * @author Jakob Bornecrantz + * @author Korcan Hussein * @ingroup oxr_api */ @@ -305,6 +306,12 @@ handle_non_null(struct oxr_instance *inst, struct oxr_logger *log, const char *n ENTRY_IF_EXT(xrGetD3D12GraphicsRequirementsKHR, KHR_D3D12_enable); #endif // OXR_HAVE_KHR_D3D12_enable +#ifdef OXR_HAVE_HTC_facial_tracking + ENTRY_IF_EXT(xrCreateFacialTrackerHTC, HTC_facial_tracking); + ENTRY_IF_EXT(xrDestroyFacialTrackerHTC, HTC_facial_tracking); + ENTRY_IF_EXT(xrGetFacialExpressionsHTC, HTC_facial_tracking); +#endif + /* * Not logging here because there's no need to loudly advertise * which extensions the loader knows about (it calls this on diff --git a/src/xrt/state_trackers/oxr/oxr_api_session.c b/src/xrt/state_trackers/oxr/oxr_api_session.c index 0242b77e2..8aefcd926 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_session.c +++ b/src/xrt/state_trackers/oxr/oxr_api_session.c @@ -1,9 +1,10 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Session entrypoints for the OpenXR state tracker. * @author Jakob Bornecrantz + * @author Korcan Hussein * @ingroup oxr_api */ @@ -678,3 +679,168 @@ oxr_xrSetAndroidApplicationThreadKHR(XrSession session, XrAndroidThreadTypeKHR t } #endif + +#ifdef OXR_HAVE_HTC_facial_tracking + +static enum xrt_facial_tracking_type_htc +oxr_to_xrt_facial_tracking_type_htc(enum XrFacialTrackingTypeHTC ft_type) +{ + return (enum xrt_facial_tracking_type_htc)ft_type; +} + +static enum xrt_input_name +oxr_facial_tracking_type_htc_to_input_name(enum xrt_facial_tracking_type_htc ft_type) +{ + switch (ft_type) { + case XRT_FACIAL_TRACKING_TYPE_LIP_DEFAULT_HTC: return XRT_INPUT_HTC_LIP_FACE_TRACKING; + case XRT_FACIAL_TRACKING_TYPE_EYE_DEFAULT_HTC: + default: return XRT_INPUT_HTC_EYE_FACE_TRACKING; + } +} + +static XrResult +oxr_facial_tracker_htc_destroy_cb(struct oxr_logger *log, struct oxr_handle_base *hb) +{ + struct oxr_facial_tracker_htc *face_tracker_htc = (struct oxr_facial_tracker_htc *)hb; + free(face_tracker_htc); + return XR_SUCCESS; +} + +XrResult +oxr_facial_tracker_htc_create(struct oxr_logger *log, + struct oxr_session *sess, + const XrFacialTrackerCreateInfoHTC *createInfo, + struct oxr_facial_tracker_htc **out_face_tracker_htc) +{ + bool supports_eye = false; + bool supports_lip = false; + oxr_system_get_face_tracking_htc_support(log, sess->sys->inst, &supports_eye, &supports_lip); + + const enum xrt_facial_tracking_type_htc facial_tracking_type = + oxr_to_xrt_facial_tracking_type_htc(createInfo->facialTrackingType); + + if (facial_tracking_type == XRT_FACIAL_TRACKING_TYPE_EYE_DEFAULT_HTC && !supports_eye) { + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "System does not support HTC eye facial tracking"); + } + if (facial_tracking_type == XRT_FACIAL_TRACKING_TYPE_LIP_DEFAULT_HTC && !supports_lip) { + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "System does not support HTC lip facial tracking"); + } + + struct xrt_device *xdev = GET_XDEV_BY_ROLE(sess->sys, face); + if (xdev == NULL) { + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "No device found for face tracking role"); + } + + if (!xdev->face_tracking_supported) { + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "Device does not support HTC facial tracking"); + } + + struct oxr_facial_tracker_htc *face_tracker_htc = NULL; + OXR_ALLOCATE_HANDLE_OR_RETURN(log, face_tracker_htc, OXR_XR_DEBUG_FTRACKER, oxr_facial_tracker_htc_destroy_cb, + &sess->handle); + + face_tracker_htc->sess = sess; + face_tracker_htc->xdev = xdev; + face_tracker_htc->facial_tracking_type = facial_tracking_type; + + *out_face_tracker_htc = face_tracker_htc; + + return XR_SUCCESS; +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrCreateFacialTrackerHTC(XrSession session, + const XrFacialTrackerCreateInfoHTC *createInfo, + XrFacialTrackerHTC *facialTracker) +{ + OXR_TRACE_MARKER(); + + struct oxr_logger log; + XrResult ret = XR_SUCCESS; + struct oxr_session *sess = NULL; + struct oxr_facial_tracker_htc *facial_tracker_htc = NULL; + OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrCreateFacialTrackerHTC"); + OXR_VERIFY_SESSION_NOT_LOST(&log, sess); + OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, createInfo, XR_TYPE_FACIAL_TRACKER_CREATE_INFO_HTC); + OXR_VERIFY_EXTENSION(&log, sess->sys->inst, HTC_facial_tracking); + + ret = oxr_facial_tracker_htc_create(&log, sess, createInfo, &facial_tracker_htc); + if (ret != XR_SUCCESS) { + return ret; + } + + OXR_VERIFY_ARG_NOT_NULL(&log, facial_tracker_htc); + *facialTracker = oxr_facial_tracker_htc_to_openxr(facial_tracker_htc); + + return XR_SUCCESS; +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrDestroyFacialTrackerHTC(XrFacialTrackerHTC facialTracker) +{ + OXR_TRACE_MARKER(); + + struct oxr_logger log; + struct oxr_facial_tracker_htc *facial_tracker_htc = NULL; + OXR_VERIFY_FACE_TRACKER_HTC_AND_INIT_LOG(&log, facialTracker, facial_tracker_htc, "xrDestroyFacialTrackerHTC"); + + return oxr_handle_destroy(&log, &facial_tracker_htc->handle); +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrGetFacialExpressionsHTC(XrFacialTrackerHTC facialTracker, XrFacialExpressionsHTC *facialExpressions) +{ + OXR_TRACE_MARKER(); + + struct oxr_logger log; + struct oxr_facial_tracker_htc *facial_tracker_htc = NULL; + OXR_VERIFY_FACE_TRACKER_HTC_AND_INIT_LOG(&log, facialExpressions, facial_tracker_htc, + "xrGetFacialExpressionsHTC"); + OXR_VERIFY_SESSION_NOT_LOST(&log, facial_tracker_htc->sess); + OXR_VERIFY_ARG_NOT_NULL(&log, facial_tracker_htc->xdev); + OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, facialExpressions, XR_TYPE_FACIAL_EXPRESSIONS_HTC); + OXR_VERIFY_ARG_NOT_NULL(&log, facialExpressions->expressionWeightings); + +#define OXR_VERIFY_FACE_EXPRESSION_COUNT(fttype) \ + if (facial_tracker_htc->facial_tracking_type == XRT_FACIAL_TRACKING_TYPE_##fttype##_DEFAULT_HTC && \ + facialExpressions->expressionCount < XRT_FACIAL_EXPRESSION_##fttype##_COUNT_HTC) { \ + return oxr_error( \ + &log, XR_ERROR_SIZE_INSUFFICIENT, \ + "\"expressionCount\" (%d) size is less than the minimum size (%d) required for " #fttype \ + " expressions.\n", \ + facialExpressions->expressionCount, XRT_FACIAL_EXPRESSION_##fttype##_COUNT_HTC); \ + } + + OXR_VERIFY_FACE_EXPRESSION_COUNT(EYE) + OXR_VERIFY_FACE_EXPRESSION_COUNT(LIP) +#undef OXR_VERIFY_FACE_EXPRESSION_COUNT + + const bool is_eye_tracking = + facial_tracker_htc->facial_tracking_type == XRT_FACIAL_TRACKING_TYPE_EYE_DEFAULT_HTC; + const size_t expression_count = + is_eye_tracking ? XRT_FACIAL_EXPRESSION_EYE_COUNT_HTC : XRT_FACIAL_EXPRESSION_LIP_COUNT_HTC; + + struct xrt_facial_expression_set facial_expression_set_result = {0}; + float *expression_weights = is_eye_tracking + ? facial_expression_set_result.eye_expression_set_htc.expression_weights + : facial_expression_set_result.lip_expression_set_htc.expression_weights; + memset(expression_weights, 0, sizeof(float) * expression_count); + + const enum xrt_input_name ft_input_name = + oxr_facial_tracking_type_htc_to_input_name(facial_tracker_htc->facial_tracking_type); + + xrt_device_get_face_tracking(facial_tracker_htc->xdev, ft_input_name, &facial_expression_set_result); + + facialExpressions->isActive = facial_expression_set_result.base_expression_set_htc.is_active; + if (facialExpressions->isActive == XR_FALSE) + return XR_SUCCESS; + + const struct oxr_instance *inst = facial_tracker_htc->sess->sys->inst; + facialExpressions->sampleTime = time_state_monotonic_to_ts_ns( + inst->timekeeping, facial_expression_set_result.base_expression_set_htc.sample_time_ns); + + memcpy(facialExpressions->expressionWeightings, expression_weights, sizeof(float) * expression_count); + + return XR_SUCCESS; +} +#endif diff --git a/src/xrt/state_trackers/oxr/oxr_api_verify.h b/src/xrt/state_trackers/oxr/oxr_api_verify.h index 517881ae9..4d1ec23f0 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_verify.h +++ b/src/xrt/state_trackers/oxr/oxr_api_verify.h @@ -1,9 +1,10 @@ -// Copyright 2018-2022, Collabora, Ltd. +// Copyright 2018-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief File for verifying app input into api functions. * @author Jakob Bornecrantz + * @author Korcan Hussein * @ingroup oxr_api */ @@ -71,6 +72,8 @@ extern "C" { OXR_VERIFY_AND_SET_AND_INIT(log, thing, new_thing, oxr_passthrough, PASSTHROUGH, name, new_thing->sess->sys->inst) #define OXR_VERIFY_PASSTHROUGH_LAYER_AND_INIT_LOG(log, thing, new_thing, name) \ OXR_VERIFY_AND_SET_AND_INIT(log, thing, new_thing, oxr_passthrough_layer, PASSTHROUGH_LAYER, name, new_thing->sess->sys->inst) +#define OXR_VERIFY_FACE_TRACKER_HTC_AND_INIT_LOG(log, thing, new_thing, name) \ + OXR_VERIFY_AND_SET_AND_INIT(log, thing, new_thing, oxr_facial_tracker_htc, FTRACKER, name, new_thing->sess->sys->inst) // clang-format on #define OXR_VERIFY_INSTANCE_NOT_NULL(log, arg, new_arg) OXR_VERIFY_SET(log, arg, new_arg, oxr_instance, INSTANCE); diff --git a/src/xrt/state_trackers/oxr/oxr_extension_support.h b/src/xrt/state_trackers/oxr/oxr_extension_support.h index 0145bab78..4a15344d1 100644 --- a/src/xrt/state_trackers/oxr/oxr_extension_support.h +++ b/src/xrt/state_trackers/oxr/oxr_extension_support.h @@ -1,4 +1,4 @@ -// Copyright 2019-2023, Collabora, Ltd. +// Copyright 2019-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -596,6 +596,17 @@ #define OXR_EXTENSION_SUPPORT_MNDX_system_buttons(_) #endif + +/* + * XR_HTC_facial_tracking + */ +#if defined(XR_HTC_facial_tracking) && defined(XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC) +#define OXR_HAVE_HTC_facial_tracking +#define OXR_EXTENSION_SUPPORT_HTC_facial_tracking(_) _(HTC_facial_tracking, HTC_FACIAL_TRACKING) +#else +#define OXR_EXTENSION_SUPPORT_HTC_facial_tracking(_) +#endif + // end of GENERATED per-extension defines - do not modify - used by scripts /*! @@ -670,5 +681,6 @@ OXR_EXTENSION_SUPPORT_MNDX_egl_enable(_) \ OXR_EXTENSION_SUPPORT_MNDX_force_feedback_curl(_) \ OXR_EXTENSION_SUPPORT_MNDX_hydra(_) \ - OXR_EXTENSION_SUPPORT_MNDX_system_buttons(_) + OXR_EXTENSION_SUPPORT_MNDX_system_buttons(_) \ + OXR_EXTENSION_SUPPORT_HTC_facial_tracking(_) // clang-format on diff --git a/src/xrt/state_trackers/oxr/oxr_instance.c b/src/xrt/state_trackers/oxr/oxr_instance.c index c5d23d25f..6fcdc1b3a 100644 --- a/src/xrt/state_trackers/oxr/oxr_instance.c +++ b/src/xrt/state_trackers/oxr/oxr_instance.c @@ -1,4 +1,4 @@ -// Copyright 2018-2023, Collabora, Ltd. +// Copyright 2018-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -283,6 +283,9 @@ oxr_instance_create(struct oxr_logger *log, #endif #ifdef OXR_HAVE_EXT_hand_interaction .ext_hand_interaction_enabled = extensions->EXT_hand_interaction, +#endif +#ifdef OXR_HAVE_HTC_facial_tracking + .htc_facial_tracking_enabled = extensions->HTC_facial_tracking, #endif }; snprintf(i_info.application_name, sizeof(inst->xinst->instance_info.application_name), "%s", diff --git a/src/xrt/state_trackers/oxr/oxr_objects.h b/src/xrt/state_trackers/oxr/oxr_objects.h index 7c8679352..575a300d9 100644 --- a/src/xrt/state_trackers/oxr/oxr_objects.h +++ b/src/xrt/state_trackers/oxr/oxr_objects.h @@ -121,6 +121,7 @@ struct oxr_interaction_profile; struct oxr_action_set_ref; struct oxr_action_ref; struct oxr_hand_tracker; +struct oxr_facial_tracker_htc; #define XRT_MAX_HANDLE_CHILDREN 256 #define OXR_MAX_BINDINGS_PER_ACTION 32 @@ -382,6 +383,18 @@ oxr_action_to_openxr(struct oxr_action *act) return XRT_CAST_PTR_TO_OXR_HANDLE(XrAction, act); } +#ifdef OXR_HAVE_HTC_facial_tracking +/*! + * To go back to a OpenXR object. + * + * @relates oxr_facial_tracker_htc + */ +static inline XrFacialTrackerHTC +oxr_facial_tracker_htc_to_openxr(struct oxr_facial_tracker_htc *face_tracker_htc) +{ + return XRT_CAST_PTR_TO_OXR_HANDLE(XrFacialTrackerHTC, face_tracker_htc); +} +#endif /*! * @@ -952,6 +965,12 @@ oxr_system_get_eye_gaze_support(struct oxr_logger *log, struct oxr_instance *ins bool oxr_system_get_force_feedback_support(struct oxr_logger *log, struct oxr_instance *inst); +void +oxr_system_get_face_tracking_htc_support(struct oxr_logger *log, + struct oxr_instance *inst, + bool *supports_eye, + bool *supports_lip); + /* * @@ -2547,6 +2566,30 @@ oxr_event_push_XrEventDataPassthroughStateChangedFB(struct oxr_logger *log, #endif // OXR_HAVE_FB_passthrough +/*! + * HTC specific Facial tracker. + * + * Parent type/handle is @ref oxr_instance + * + * + * @obj{XrFacialTrackerHTC} + * @extends oxr_handle_base + */ +struct oxr_facial_tracker_htc +{ + //! Common structure for things referred to by OpenXR handles. + struct oxr_handle_base handle; + + //! Owner of this face tracker. + struct oxr_session *sess; + + //! xrt_device backing this face tracker + struct xrt_device *xdev; + + //! Type of facial tracking, eyes or lips + enum xrt_facial_tracking_type_htc facial_tracking_type; +}; + /*! * @} */ diff --git a/src/xrt/state_trackers/oxr/oxr_session.c b/src/xrt/state_trackers/oxr/oxr_session.c index f7d7c7e01..dd4cbf300 100644 --- a/src/xrt/state_trackers/oxr/oxr_session.c +++ b/src/xrt/state_trackers/oxr/oxr_session.c @@ -1,4 +1,4 @@ -// Copyright 2018-2023, Collabora, Ltd. +// Copyright 2018-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -228,6 +228,9 @@ oxr_session_begin(struct oxr_logger *log, struct oxr_session *sess, const XrSess #endif #ifdef OXR_HAVE_EXT_hand_interaction .ext_hand_interaction_enabled = extensions->EXT_hand_interaction, +#endif +#ifdef OXR_HAVE_HTC_facial_tracking + .htc_facial_tracking_enabled = extensions->HTC_facial_tracking, #endif }; diff --git a/src/xrt/state_trackers/oxr/oxr_system.c b/src/xrt/state_trackers/oxr/oxr_system.c index 706c66328..47141e52b 100644 --- a/src/xrt/state_trackers/oxr/oxr_system.c +++ b/src/xrt/state_trackers/oxr/oxr_system.c @@ -1,9 +1,10 @@ -// Copyright 2018-2020, Collabora, Ltd. +// Copyright 2018-2024, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Holds system related entrypoints. * @author Jakob Bornecrantz + * @author Korcan Hussein * @ingroup oxr_main */ @@ -269,6 +270,35 @@ oxr_system_get_force_feedback_support(struct oxr_logger *log, struct oxr_instanc return left_supported || right_supported; } +void +oxr_system_get_face_tracking_htc_support(struct oxr_logger *log, + struct oxr_instance *inst, + bool *supports_eye, + bool *supports_lip) +{ + struct oxr_system *sys = &inst->system; + struct xrt_device *face_xdev = GET_XDEV_BY_ROLE(sys, face); + + if (supports_eye) + *supports_eye = false; + if (supports_lip) + *supports_lip = false; + + if (face_xdev == NULL || !face_xdev->face_tracking_supported || face_xdev->inputs == NULL) { + return; + } + + for (size_t input_idx = 0; input_idx < face_xdev->input_count; ++input_idx) { + const struct xrt_input *input = &face_xdev->inputs[input_idx]; + if (supports_eye != NULL && input->name == XRT_INPUT_HTC_EYE_FACE_TRACKING) { + *supports_eye = true; + } + if (supports_lip != NULL && input->name == XRT_INPUT_HTC_LIP_FACE_TRACKING) { + *supports_eye = true; + } + } +} + XrResult oxr_system_get_properties(struct oxr_logger *log, struct oxr_system *sys, XrSystemProperties *properties) { @@ -347,6 +377,21 @@ oxr_system_get_properties(struct oxr_logger *log, struct oxr_system *sys, XrSyst } #endif +#ifdef OXR_HAVE_HTC_facial_tracking + XrSystemFacialTrackingPropertiesHTC *htc_facial_tracking_props = NULL; + if (sys->inst->extensions.HTC_facial_tracking) { + htc_facial_tracking_props = OXR_GET_OUTPUT_FROM_CHAIN( + properties, XR_TYPE_SYSTEM_FACIAL_TRACKING_PROPERTIES_HTC, XrSystemFacialTrackingPropertiesHTC); + } + + if (htc_facial_tracking_props) { + bool supports_eye = false; + bool supports_lip = false; + oxr_system_get_face_tracking_htc_support(log, sys->inst, &supports_eye, &supports_lip); + htc_facial_tracking_props->supportEyeFacialTracking = supports_eye; + htc_facial_tracking_props->supportLipFacialTracking = supports_lip; + } +#endif // OXR_HAVE_HTC_facial_tracking return XR_SUCCESS; }