diff --git a/src/xrt/drivers/ht/ht_driver.c b/src/xrt/drivers/ht/ht_driver.c index 35f47efbf..08787a596 100644 --- a/src/xrt/drivers/ht/ht_driver.c +++ b/src/xrt/drivers/ht/ht_driver.c @@ -234,7 +234,7 @@ ht_device_create_common(struct t_stereo_camera_calibration *calib, int ht_device_create(struct xrt_frame_context *xfctx, struct t_stereo_camera_calibration *calib, - struct t_camera_extra_info extra_camera_info, + struct t_hand_tracking_create_info create_info, struct xrt_slam_sinks **out_sinks, struct xrt_device **out_device) { @@ -254,7 +254,7 @@ ht_device_create(struct xrt_frame_context *xfctx, return -1; } - sync = t_hand_tracking_sync_mercury_create(calib, extra_camera_info, path); + sync = t_hand_tracking_sync_mercury_create(calib, create_info, path); struct ht_device *htd = ht_device_create_common(calib, false, xfctx, sync); diff --git a/src/xrt/drivers/ht/ht_interface.h b/src/xrt/drivers/ht/ht_interface.h index bd0d828bd..0121bd9cd 100644 --- a/src/xrt/drivers/ht/ht_interface.h +++ b/src/xrt/drivers/ht/ht_interface.h @@ -16,6 +16,7 @@ #include "tracking/t_tracking.h" #include "tracking/t_hand_tracking.h" #include "xrt/xrt_prober.h" +#include "xrt/xrt_tracking.h" #ifdef __cplusplus @@ -36,6 +37,7 @@ extern "C" { * * @param xfctx Frame context to attach the tracker to * @param calib Calibration struct for stereo camera + * @param create_info Additional creation options * @param out_sinks Sinks to stream camera data to * @param out_device Newly created hand tracker "device" * @return int 0 on success @@ -43,7 +45,7 @@ extern "C" { int ht_device_create(struct xrt_frame_context *xfctx, struct t_stereo_camera_calibration *calib, - struct t_camera_extra_info extra_camera_info, + struct t_hand_tracking_create_info create_info, struct xrt_slam_sinks **out_sinks, struct xrt_device **out_device); diff --git a/src/xrt/drivers/rift_s/rift_s_tracker.c b/src/xrt/drivers/rift_s/rift_s_tracker.c index d19ee28aa..51d504b9b 100644 --- a/src/xrt/drivers/rift_s/rift_s_tracker.c +++ b/src/xrt/drivers/rift_s/rift_s_tracker.c @@ -252,10 +252,13 @@ rift_s_create_hand_tracker(struct rift_s_tracker *t, extra_camera_info.views[0].camera_orientation = CAMERA_ORIENTATION_90; extra_camera_info.views[1].camera_orientation = CAMERA_ORIENTATION_90; + // TODO@mateosss: Use proper masks_sink in all drivers and not null + struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = NULL}; + int create_status = ht_device_create(xfctx, // t->stereo_calib, // - extra_camera_info, - &sinks, // + create_info, // + &sinks, // &device); if (create_status != 0) { return create_status; diff --git a/src/xrt/drivers/wmr/wmr_hmd.c b/src/xrt/drivers/wmr/wmr_hmd.c index b78c2e024..8fbd5d952 100644 --- a/src/xrt/drivers/wmr/wmr_hmd.c +++ b/src/xrt/drivers/wmr/wmr_hmd.c @@ -1651,6 +1651,7 @@ wmr_hmd_guess_camera_orientation(struct wmr_hmd *wh) static int wmr_hmd_hand_track(struct wmr_hmd *wh, struct t_stereo_camera_calibration *stereo_calib, + struct xrt_hand_masks_sink *masks_sink, struct xrt_slam_sinks **out_sinks, struct xrt_device **out_device) { @@ -1674,12 +1675,15 @@ wmr_hmd_hand_track(struct wmr_hmd *wh, //!@todo Turning it off is okay for now, but we should plug metric_radius (or whatever it's called) in, at some //! point. + // TODO@mateosss: do it now extra_camera_info.views[0].boundary_type = HT_IMAGE_BOUNDARY_NONE; extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE; + struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = masks_sink}; + int create_status = ht_device_create(&wh->tracking.xfctx, // stereo_calib, // - extra_camera_info, // + create_info, // &sinks, // &device); if (create_status != 0) { @@ -1819,8 +1823,9 @@ wmr_hmd_setup_trackers(struct wmr_hmd *wh, struct xrt_slam_sinks *out_sinks, str // Initialize hand tracker struct xrt_slam_sinks *hand_sinks = NULL; struct xrt_device *hand_device = NULL; + struct xrt_hand_masks_sink *masks_sink = slam_sinks->hand_masks; if (wh->tracking.hand_enabled) { - int hand_status = wmr_hmd_hand_track(wh, stereo_calib, &hand_sinks, &hand_device); + int hand_status = wmr_hmd_hand_track(wh, stereo_calib, masks_sink, &hand_sinks, &hand_device); if (hand_status != 0 || hand_sinks == NULL || hand_device == NULL) { WMR_WARN(wh, "Unable to setup the hand tracker"); return false; diff --git a/src/xrt/include/tracking/t_hand_tracking.h b/src/xrt/include/tracking/t_hand_tracking.h index 0e6c5694d..ac75bcfbf 100644 --- a/src/xrt/include/tracking/t_hand_tracking.h +++ b/src/xrt/include/tracking/t_hand_tracking.h @@ -104,6 +104,15 @@ struct t_camera_extra_info struct t_camera_extra_info_one_view views[2]; }; +/*! + * Creation info for the creation of a hand tracker + */ +struct t_hand_tracking_create_info +{ + struct t_camera_extra_info cams_info; //!< Extra camera info + struct xrt_hand_masks_sink *masks_sink; //!< Optional sink to stream hand bounding boxes to +}; + /*! * Synchronously processes frames and returns two hands. */ diff --git a/src/xrt/state_trackers/gui/gui_scene_hand_tracking_demo.c b/src/xrt/state_trackers/gui/gui_scene_hand_tracking_demo.c index 0e6b457b9..b769ad4ee 100644 --- a/src/xrt/state_trackers/gui/gui_scene_hand_tracking_demo.c +++ b/src/xrt/state_trackers/gui/gui_scene_hand_tracking_demo.c @@ -77,10 +77,12 @@ gui_scene_hand_tracking_demo(struct gui_program *p) extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE; extra_camera_info.views[1].camera_orientation = CAMERA_ORIENTATION_0; + struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = NULL}; + int create_status = ht_device_create( // &usysd->xfctx, // calib, // - extra_camera_info, // + create_info, // &hand_sinks, // &ht_dev); // t_stereo_camera_calibration_reference(&calib, NULL); diff --git a/src/xrt/targets/common/target_builder_lighthouse.c b/src/xrt/targets/common/target_builder_lighthouse.c index 889f30dea..fb522230d 100644 --- a/src/xrt/targets/common/target_builder_lighthouse.c +++ b/src/xrt/targets/common/target_builder_lighthouse.c @@ -233,11 +233,13 @@ valve_index_hand_track(struct lighthouse_system *lhs, info.views[0].boundary.circle.normalized_radius = 0.55; info.views[1].boundary.circle.normalized_radius = 0.55; + struct t_hand_tracking_create_info create_info = {.cams_info = info, .masks_sink = NULL}; + struct xrt_device *ht_device = NULL; int create_status = ht_device_create( // xfctx, // stereo_calib, // - info, // + create_info, // &sinks, // &ht_device); if (create_status != 0) { diff --git a/src/xrt/targets/common/target_builder_north_star.c b/src/xrt/targets/common/target_builder_north_star.c index b453aa237..7ea4c4aa4 100644 --- a/src/xrt/targets/common/target_builder_north_star.c +++ b/src/xrt/targets/common/target_builder_north_star.c @@ -311,10 +311,12 @@ ns_setup_depthai_device(struct ns_builder *nsb, extra_camera_info.views[0].boundary_type = HT_IMAGE_BOUNDARY_NONE; extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE; + struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = NULL}; + int create_status = ht_device_create( // xfctx, // calib, // - extra_camera_info, // + create_info, // &hand_sinks, // out_hand_device); // t_stereo_camera_calibration_reference(&calib, NULL); diff --git a/src/xrt/tracking/hand/mercury/hg_interface.h b/src/xrt/tracking/hand/mercury/hg_interface.h index 2351e7a4d..cb08586f3 100644 --- a/src/xrt/tracking/hand/mercury/hg_interface.h +++ b/src/xrt/tracking/hand/mercury/hg_interface.h @@ -23,7 +23,7 @@ extern "C" { */ struct t_hand_tracking_sync * t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib, - struct t_camera_extra_info extra_camera_info, + struct t_hand_tracking_create_info create_info, const char *models_folder); #ifdef __cplusplus diff --git a/src/xrt/tracking/hand/mercury/hg_sync.cpp b/src/xrt/tracking/hand/mercury/hg_sync.cpp index 1a7d893eb..d2cf25869 100644 --- a/src/xrt/tracking/hand/mercury/hg_sync.cpp +++ b/src/xrt/tracking/hand/mercury/hg_sync.cpp @@ -11,11 +11,14 @@ #include "hg_sync.hpp" #include "hg_image_math.inl" +#include "tracking/t_hand_tracking.h" #include "util/u_box_iou.hpp" #include "util/u_hand_tracking.h" #include "math/m_vec2.h" #include "util/u_misc.h" +#include "xrt/xrt_defines.h" #include "xrt/xrt_frame.h" +#include "xrt/xrt_tracking.h" #include @@ -509,6 +512,8 @@ void predict_new_regions_of_interest(struct HandTracking *hgt) { + xrt_hand_masks_sample masks{}; // Zero initialization + for (int hand_idx = 0; hand_idx < 2; hand_idx++) { // If we don't have the past two frames, this code doesn't do what we want. // If we only have *one* frame, we just reuse the same bounding box and hope the hand @@ -557,16 +562,33 @@ predict_new_regions_of_interest(struct HandTracking *hgt) num_outside); for (int view_idx = 0; view_idx < 2; view_idx++) { - if (num_outside[view_idx] < hgt->tuneable_values.max_num_outside_view) { - hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].provenance = - ROIProvenance::POSE_PREDICTION; - hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].found = true; + hand_region_of_interest &hroi = hgt->views[view_idx].regions_of_interest_this_frame[hand_idx]; + auto &masks_view = masks.views[view_idx]; + auto &masks_hand = masks_view.hands[hand_idx]; + masks_view.enabled = true; + + if (num_outside[view_idx] < hgt->tuneable_values.max_num_outside_view) { + hroi.provenance = ROIProvenance::POSE_PREDICTION; + hroi.found = true; + + xrt_vec2 ¢er = hroi.center_px; + float &size = hroi.size_px; + masks_hand.rect.offset.w = int(center.x - size / 2); + masks_hand.rect.offset.h = int(center.y - size / 2); + masks_hand.rect.extent.w = int(size); + masks_hand.rect.extent.h = int(size); + masks_hand.enabled = true; } else { - hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].found = false; + hroi.found = false; + masks_hand.enabled = false; } } } + + if (hgt->hand_masks_sink != NULL) { + xrt_sink_push_hand_masks(hgt->hand_masks_sink, &masks); + } } //!@todo This looks like it sucks, but it doesn't given the current architecture. @@ -1070,7 +1092,7 @@ using namespace xrt::tracking::hand::mercury; extern "C" t_hand_tracking_sync * t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib, - struct t_camera_extra_info extra_camera_info, + struct t_hand_tracking_create_info create_info, const char *models_folder) { XRT_TRACE_MARKER(); @@ -1095,6 +1117,9 @@ t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib, hgt->views[0].hgt = hgt; hgt->views[1].hgt = hgt; // :) + hgt->hand_masks_sink = create_info.masks_sink; + + struct t_camera_extra_info &extra_camera_info = create_info.cams_info; hgt->views[0].camera_info = extra_camera_info.views[0]; hgt->views[1].camera_info = extra_camera_info.views[1]; diff --git a/src/xrt/tracking/hand/mercury/hg_sync.hpp b/src/xrt/tracking/hand/mercury/hg_sync.hpp index 543ee2d17..e9ce8b2ff 100644 --- a/src/xrt/tracking/hand/mercury/hg_sync.hpp +++ b/src/xrt/tracking/hand/mercury/hg_sync.hpp @@ -18,6 +18,7 @@ #include "xrt/xrt_defines.h" #include "xrt/xrt_frame.h" +#include "xrt/xrt_tracking.h" #include "math/m_api.h" #include "math/m_vec2.h" @@ -253,6 +254,7 @@ public: struct u_sink_debug debug_sink_ann = {}; struct u_sink_debug debug_sink_model = {}; + struct xrt_hand_masks_sink *hand_masks_sink; float multiply_px_coord_for_undistort;