h/mercury: Push hand rect masks to the SLAM tracker sinks

src/xrt/drivers/ht/ht_driver.c

@@ -234,7 +234,7 @@ ht_device_create_common(struct t_stereo_camera_calibration *calib,
 int
 ht_device_create(struct xrt_frame_context *xfctx,
                  struct t_stereo_camera_calibration *calib,
-                 struct t_camera_extra_info extra_camera_info,
+                 struct t_hand_tracking_create_info create_info,
                  struct xrt_slam_sinks **out_sinks,
                  struct xrt_device **out_device)
 {
@@ -254,7 +254,7 @@ ht_device_create(struct xrt_frame_context *xfctx,
 		return -1;
 	}
 
-	sync = t_hand_tracking_sync_mercury_create(calib, extra_camera_info, path);
+	sync = t_hand_tracking_sync_mercury_create(calib, create_info, path);
 
 	struct ht_device *htd = ht_device_create_common(calib, false, xfctx, sync);
 

src/xrt/drivers/ht/ht_interface.h

@@ -16,6 +16,7 @@
 #include "tracking/t_tracking.h"
 #include "tracking/t_hand_tracking.h"
 #include "xrt/xrt_prober.h"
+#include "xrt/xrt_tracking.h"
 
 
 #ifdef __cplusplus
@@ -36,6 +37,7 @@ extern "C" {
  *
  * @param xfctx Frame context to attach the tracker to
  * @param calib Calibration struct for stereo camera
+ * @param create_info Additional creation options
  * @param out_sinks Sinks to stream camera data to
  * @param out_device Newly created hand tracker "device"
  * @return int 0 on success
@@ -43,7 +45,7 @@ extern "C" {
 int
 ht_device_create(struct xrt_frame_context *xfctx,
                  struct t_stereo_camera_calibration *calib,
-                 struct t_camera_extra_info extra_camera_info,
+                 struct t_hand_tracking_create_info create_info,
                  struct xrt_slam_sinks **out_sinks,
                  struct xrt_device **out_device);
 

src/xrt/drivers/rift_s/rift_s_tracker.c

@@ -252,10 +252,13 @@ rift_s_create_hand_tracker(struct rift_s_tracker *t,
 	extra_camera_info.views[0].camera_orientation = CAMERA_ORIENTATION_90;
 	extra_camera_info.views[1].camera_orientation = CAMERA_ORIENTATION_90;
 
+	// TODO@mateosss: Use proper masks_sink in all drivers and not null
+	struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = NULL};
+
 	int create_status = ht_device_create(xfctx,           //
 	                                     t->stereo_calib, //
-	                                     extra_camera_info,
+	                                     create_info,     //
-	                                     &sinks, //
+	                                     &sinks,          //
 	                                     &device);
 	if (create_status != 0) {
 		return create_status;

src/xrt/drivers/wmr/wmr_hmd.c

@@ -1651,6 +1651,7 @@ wmr_hmd_guess_camera_orientation(struct wmr_hmd *wh)
 static int
 wmr_hmd_hand_track(struct wmr_hmd *wh,
                    struct t_stereo_camera_calibration *stereo_calib,
+                   struct xrt_hand_masks_sink *masks_sink,
                    struct xrt_slam_sinks **out_sinks,
                    struct xrt_device **out_device)
 {
@@ -1674,12 +1675,15 @@ wmr_hmd_hand_track(struct wmr_hmd *wh,
 
 	//!@todo Turning it off is okay for now, but we should plug metric_radius (or whatever it's called) in, at some
 	//! point.
+	// TODO@mateosss: do it now
 	extra_camera_info.views[0].boundary_type = HT_IMAGE_BOUNDARY_NONE;
 	extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE;
 
+	struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = masks_sink};
+
 	int create_status = ht_device_create(&wh->tracking.xfctx, //
 	                                     stereo_calib,        //
-	                                     extra_camera_info,   //
+	                                     create_info,         //
 	                                     &sinks,              //
 	                                     &device);
 	if (create_status != 0) {
@@ -1819,8 +1823,9 @@ wmr_hmd_setup_trackers(struct wmr_hmd *wh, struct xrt_slam_sinks *out_sinks, str
 	// Initialize hand tracker
 	struct xrt_slam_sinks *hand_sinks = NULL;
 	struct xrt_device *hand_device = NULL;
+	struct xrt_hand_masks_sink *masks_sink = slam_sinks->hand_masks;
 	if (wh->tracking.hand_enabled) {
-		int hand_status = wmr_hmd_hand_track(wh, stereo_calib, &hand_sinks, &hand_device);
+		int hand_status = wmr_hmd_hand_track(wh, stereo_calib, masks_sink, &hand_sinks, &hand_device);
 		if (hand_status != 0 || hand_sinks == NULL || hand_device == NULL) {
 			WMR_WARN(wh, "Unable to setup the hand tracker");
 			return false;

src/xrt/include/tracking/t_hand_tracking.h

@@ -104,6 +104,15 @@ struct t_camera_extra_info
 	struct t_camera_extra_info_one_view views[2];
 };
 
+/*!
+ * Creation info for the creation of a hand tracker
+ */
+struct t_hand_tracking_create_info
+{
+	struct t_camera_extra_info cams_info;   //!< Extra camera info
+	struct xrt_hand_masks_sink *masks_sink; //!< Optional sink to stream hand bounding boxes to
+};
+
 /*!
  * Synchronously processes frames and returns two hands.
  */

src/xrt/state_trackers/gui/gui_scene_hand_tracking_demo.c

@@ -77,10 +77,12 @@ gui_scene_hand_tracking_demo(struct gui_program *p)
 	extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE;
 	extra_camera_info.views[1].camera_orientation = CAMERA_ORIENTATION_0;
 
+	struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = NULL};
+
 	int create_status = ht_device_create( //
 	    &usysd->xfctx,                    //
 	    calib,                            //
-	    extra_camera_info,                //
+	    create_info,                      //
 	    &hand_sinks,                      //
 	    &ht_dev);                         //
 	t_stereo_camera_calibration_reference(&calib, NULL);

src/xrt/targets/common/target_builder_lighthouse.c

@@ -233,11 +233,13 @@ valve_index_hand_track(struct lighthouse_system *lhs,
 	info.views[0].boundary.circle.normalized_radius = 0.55;
 	info.views[1].boundary.circle.normalized_radius = 0.55;
 
+	struct t_hand_tracking_create_info create_info = {.cams_info = info, .masks_sink = NULL};
+
 	struct xrt_device *ht_device = NULL;
 	int create_status = ht_device_create( //
 	    xfctx,                            //
 	    stereo_calib,                     //
-	    info,                             //
+	    create_info,                      //
 	    &sinks,                           //
 	    &ht_device);
 	if (create_status != 0) {

src/xrt/targets/common/target_builder_north_star.c

@@ -311,10 +311,12 @@ ns_setup_depthai_device(struct ns_builder *nsb,
 	extra_camera_info.views[0].boundary_type = HT_IMAGE_BOUNDARY_NONE;
 	extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE;
 
+	struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = NULL};
+
 	int create_status = ht_device_create( //
 	    xfctx,                            //
 	    calib,                            //
-	    extra_camera_info,                //
+	    create_info,                      //
 	    &hand_sinks,                      //
 	    out_hand_device);                 //
 	t_stereo_camera_calibration_reference(&calib, NULL);

src/xrt/tracking/hand/mercury/hg_interface.h

@@ -23,7 +23,7 @@ extern "C" {
  */
 struct t_hand_tracking_sync *
 t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib,
-                                    struct t_camera_extra_info extra_camera_info,
+                                    struct t_hand_tracking_create_info create_info,
                                     const char *models_folder);
 
 #ifdef __cplusplus

src/xrt/tracking/hand/mercury/hg_sync.cpp

@@ -11,11 +11,14 @@
 
 #include "hg_sync.hpp"
 #include "hg_image_math.inl"
+#include "tracking/t_hand_tracking.h"
 #include "util/u_box_iou.hpp"
 #include "util/u_hand_tracking.h"
 #include "math/m_vec2.h"
 #include "util/u_misc.h"
+#include "xrt/xrt_defines.h"
 #include "xrt/xrt_frame.h"
+#include "xrt/xrt_tracking.h"
 
 
 #include <numeric>
@@ -509,6 +512,8 @@ void
 predict_new_regions_of_interest(struct HandTracking *hgt)
 {
 
+	xrt_hand_masks_sample masks{}; // Zero initialization
+
 	for (int hand_idx = 0; hand_idx < 2; hand_idx++) {
 		// If we don't have the past two frames, this code doesn't do what we want.
 		// If we only have *one* frame, we just reuse the same bounding box and hope the hand
@@ -557,16 +562,33 @@ predict_new_regions_of_interest(struct HandTracking *hgt)
 		             num_outside);
 
 		for (int view_idx = 0; view_idx < 2; view_idx++) {
-			if (num_outside[view_idx] < hgt->tuneable_values.max_num_outside_view) {
+			hand_region_of_interest &hroi = hgt->views[view_idx].regions_of_interest_this_frame[hand_idx];
-				hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].provenance =
+			auto &masks_view = masks.views[view_idx];
-				    ROIProvenance::POSE_PREDICTION;
+			auto &masks_hand = masks_view.hands[hand_idx];
-				hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].found = true;
 
+			masks_view.enabled = true;
+
+			if (num_outside[view_idx] < hgt->tuneable_values.max_num_outside_view) {
+				hroi.provenance = ROIProvenance::POSE_PREDICTION;
+				hroi.found = true;
+
+				xrt_vec2 &center = hroi.center_px;
+				float &size = hroi.size_px;
+				masks_hand.rect.offset.w = int(center.x - size / 2);
+				masks_hand.rect.offset.h = int(center.y - size / 2);
+				masks_hand.rect.extent.w = int(size);
+				masks_hand.rect.extent.h = int(size);
+				masks_hand.enabled = true;
 			} else {
-				hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].found = false;
+				hroi.found = false;
+				masks_hand.enabled = false;
 			}
 		}
 	}
+
+	if (hgt->hand_masks_sink != NULL) {
+		xrt_sink_push_hand_masks(hgt->hand_masks_sink, &masks);
+	}
 }
 
 //!@todo This looks like it sucks, but it doesn't given the current architecture.
@@ -1070,7 +1092,7 @@ using namespace xrt::tracking::hand::mercury;
 
 extern "C" t_hand_tracking_sync *
 t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib,
-                                    struct t_camera_extra_info extra_camera_info,
+                                    struct t_hand_tracking_create_info create_info,
                                     const char *models_folder)
 {
 	XRT_TRACE_MARKER();
@@ -1095,6 +1117,9 @@ t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib,
 	hgt->views[0].hgt = hgt;
 	hgt->views[1].hgt = hgt; // :)
 
+	hgt->hand_masks_sink = create_info.masks_sink;
+
+	struct t_camera_extra_info &extra_camera_info = create_info.cams_info;
 	hgt->views[0].camera_info = extra_camera_info.views[0];
 	hgt->views[1].camera_info = extra_camera_info.views[1];
 

src/xrt/tracking/hand/mercury/hg_sync.hpp

@@ -18,6 +18,7 @@
 
 #include "xrt/xrt_defines.h"
 #include "xrt/xrt_frame.h"
+#include "xrt/xrt_tracking.h"
 
 #include "math/m_api.h"
 #include "math/m_vec2.h"
@@ -253,6 +254,7 @@ public:
 
 	struct u_sink_debug debug_sink_ann = {};
 	struct u_sink_debug debug_sink_model = {};
+	struct xrt_hand_masks_sink *hand_masks_sink;
 
 	float multiply_px_coord_for_undistort;