mirrors/monado
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/monado/monado.git synced 2024-12-29 11:06:18 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

h/mercury: Push hand rect masks to the SLAM tracker sinks

Browse source
This commit is contained in:
Mateo de Mayo 2024-01-23 14:12:41 -03:00 committed by Simon Zeni
parent 4cfe58a5e9
commit e79d2d395f
11 changed files with 69 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/xrt/drivers/ht/ht_driver.c
View file

@ -234,7 +234,7 @@ ht_device_create_common(struct t_stereo_camera_calibration *calib,
int
ht_device_create(struct xrt_frame_context *xfctx,
struct t_stereo_camera_calibration *calib,
struct t_camera_extra_info extra_camera_info,
struct t_hand_tracking_create_info create_info,
struct xrt_slam_sinks **out_sinks,
struct xrt_device **out_device)
{
@ -254,7 +254,7 @@ ht_device_create(struct xrt_frame_context *xfctx,
return -1;
}
sync = t_hand_tracking_sync_mercury_create(calib, extra_camera_info, path);
sync = t_hand_tracking_sync_mercury_create(calib, create_info, path);
struct ht_device *htd = ht_device_create_common(calib, false, xfctx, sync);

4
src/xrt/drivers/ht/ht_interface.h
View file

@ -16,6 +16,7 @@
#include "tracking/t_tracking.h"
#include "tracking/t_hand_tracking.h"
#include "xrt/xrt_prober.h"
#include "xrt/xrt_tracking.h"
#ifdef __cplusplus
@ -36,6 +37,7 @@ extern "C" {
*
* @param xfctx Frame context to attach the tracker to
* @param calib Calibration struct for stereo camera
* @param create_info Additional creation options
* @param out_sinks Sinks to stream camera data to
* @param out_device Newly created hand tracker "device"
* @return int 0 on success
@ -43,7 +45,7 @@ extern "C" {
int
ht_device_create(struct xrt_frame_context *xfctx,
struct t_stereo_camera_calibration *calib,
struct t_camera_extra_info extra_camera_info,
struct t_hand_tracking_create_info create_info,
struct xrt_slam_sinks **out_sinks,
struct xrt_device **out_device);

5
src/xrt/drivers/rift_s/rift_s_tracker.c
View file

@ -252,9 +252,12 @@ rift_s_create_hand_tracker(struct rift_s_tracker *t,
extra_camera_info.views[0].camera_orientation = CAMERA_ORIENTATION_90;
extra_camera_info.views[1].camera_orientation = CAMERA_ORIENTATION_90;
// TODO@mateosss: Use proper masks_sink in all drivers and not null
struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = NULL};
int create_status = ht_device_create(xfctx, //
t->stereo_calib, //
extra_camera_info,
create_info, //
&sinks, //
&device);
if (create_status != 0) {

9
src/xrt/drivers/wmr/wmr_hmd.c
View file

@ -1651,6 +1651,7 @@ wmr_hmd_guess_camera_orientation(struct wmr_hmd *wh)
static int
wmr_hmd_hand_track(struct wmr_hmd *wh,
struct t_stereo_camera_calibration *stereo_calib,
struct xrt_hand_masks_sink *masks_sink,
struct xrt_slam_sinks **out_sinks,
struct xrt_device **out_device)
{
@ -1674,12 +1675,15 @@ wmr_hmd_hand_track(struct wmr_hmd *wh,
//!@todo Turning it off is okay for now, but we should plug metric_radius (or whatever it's called) in, at some
//! point.
// TODO@mateosss: do it now
extra_camera_info.views[0].boundary_type = HT_IMAGE_BOUNDARY_NONE;
extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE;
struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = masks_sink};
int create_status = ht_device_create(&wh->tracking.xfctx, //
stereo_calib, //
extra_camera_info, //
create_info, //
&sinks, //
&device);
if (create_status != 0) {
@ -1819,8 +1823,9 @@ wmr_hmd_setup_trackers(struct wmr_hmd *wh, struct xrt_slam_sinks *out_sinks, str
// Initialize hand tracker
struct xrt_slam_sinks *hand_sinks = NULL;
struct xrt_device *hand_device = NULL;
struct xrt_hand_masks_sink *masks_sink = slam_sinks->hand_masks;
if (wh->tracking.hand_enabled) {
int hand_status = wmr_hmd_hand_track(wh, stereo_calib, &hand_sinks, &hand_device);
int hand_status = wmr_hmd_hand_track(wh, stereo_calib, masks_sink, &hand_sinks, &hand_device);
if (hand_status != 0 || hand_sinks == NULL || hand_device == NULL) {
WMR_WARN(wh, "Unable to setup the hand tracker");
return false;

9
src/xrt/include/tracking/t_hand_tracking.h
View file

@ -104,6 +104,15 @@ struct t_camera_extra_info
struct t_camera_extra_info_one_view views[2];
};
/*!
* Creation info for the creation of a hand tracker
*/
struct t_hand_tracking_create_info
{
struct t_camera_extra_info cams_info; //!< Extra camera info
struct xrt_hand_masks_sink *masks_sink; //!< Optional sink to stream hand bounding boxes to
};
/*!
* Synchronously processes frames and returns two hands.
*/

4
src/xrt/state_trackers/gui/gui_scene_hand_tracking_demo.c
View file

@ -77,10 +77,12 @@ gui_scene_hand_tracking_demo(struct gui_program *p)
extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE;
extra_camera_info.views[1].camera_orientation = CAMERA_ORIENTATION_0;
struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = NULL};
int create_status = ht_device_create( //
&usysd->xfctx, //
calib, //
extra_camera_info, //
create_info, //
&hand_sinks, //
&ht_dev); //
t_stereo_camera_calibration_reference(&calib, NULL);

4
src/xrt/targets/common/target_builder_lighthouse.c
View file

@ -233,11 +233,13 @@ valve_index_hand_track(struct lighthouse_system *lhs,
info.views[0].boundary.circle.normalized_radius = 0.55;
info.views[1].boundary.circle.normalized_radius = 0.55;
struct t_hand_tracking_create_info create_info = {.cams_info = info, .masks_sink = NULL};
struct xrt_device *ht_device = NULL;
int create_status = ht_device_create( //
xfctx, //
stereo_calib, //
info, //
create_info, //
&sinks, //
&ht_device);
if (create_status != 0) {

4
src/xrt/targets/common/target_builder_north_star.c
View file

@ -311,10 +311,12 @@ ns_setup_depthai_device(struct ns_builder *nsb,
extra_camera_info.views[0].boundary_type = HT_IMAGE_BOUNDARY_NONE;
extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE;
struct t_hand_tracking_create_info create_info = {.cams_info = extra_camera_info, .masks_sink = NULL};
int create_status = ht_device_create( //
xfctx, //
calib, //
extra_camera_info, //
create_info, //
&hand_sinks, //
out_hand_device); //
t_stereo_camera_calibration_reference(&calib, NULL);

2
src/xrt/tracking/hand/mercury/hg_interface.h
View file

@ -23,7 +23,7 @@ extern "C" {
*/
struct t_hand_tracking_sync *
t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib,
struct t_camera_extra_info extra_camera_info,
struct t_hand_tracking_create_info create_info,
const char *models_folder);
#ifdef __cplusplus

37
src/xrt/tracking/hand/mercury/hg_sync.cpp
View file

@ -11,11 +11,14 @@
#include "hg_sync.hpp"
#include "hg_image_math.inl"
#include "tracking/t_hand_tracking.h"
#include "util/u_box_iou.hpp"
#include "util/u_hand_tracking.h"
#include "math/m_vec2.h"
#include "util/u_misc.h"
#include "xrt/xrt_defines.h"
#include "xrt/xrt_frame.h"
#include "xrt/xrt_tracking.h"
#include <numeric>
@ -509,6 +512,8 @@ void
predict_new_regions_of_interest(struct HandTracking *hgt)
{
xrt_hand_masks_sample masks{}; // Zero initialization
for (int hand_idx = 0; hand_idx < 2; hand_idx++) {
// If we don't have the past two frames, this code doesn't do what we want.
// If we only have *one* frame, we just reuse the same bounding box and hope the hand
@ -557,16 +562,33 @@ predict_new_regions_of_interest(struct HandTracking *hgt)
num_outside);
for (int view_idx = 0; view_idx < 2; view_idx++) {
if (num_outside[view_idx] < hgt->tuneable_values.max_num_outside_view) {
hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].provenance =
ROIProvenance::POSE_PREDICTION;
hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].found = true;
hand_region_of_interest &hroi = hgt->views[view_idx].regions_of_interest_this_frame[hand_idx];
auto &masks_view = masks.views[view_idx];
auto &masks_hand = masks_view.hands[hand_idx];
masks_view.enabled = true;
if (num_outside[view_idx] < hgt->tuneable_values.max_num_outside_view) {
hroi.provenance = ROIProvenance::POSE_PREDICTION;
hroi.found = true;
xrt_vec2 &center = hroi.center_px;
float &size = hroi.size_px;
masks_hand.rect.offset.w = int(center.x - size / 2);
masks_hand.rect.offset.h = int(center.y - size / 2);
masks_hand.rect.extent.w = int(size);
masks_hand.rect.extent.h = int(size);
masks_hand.enabled = true;
} else {
hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].found = false;
hroi.found = false;
masks_hand.enabled = false;
}
}
}
if (hgt->hand_masks_sink != NULL) {
xrt_sink_push_hand_masks(hgt->hand_masks_sink, &masks);
}
}
//!@todo This looks like it sucks, but it doesn't given the current architecture.
@ -1070,7 +1092,7 @@ using namespace xrt::tracking::hand::mercury;
extern "C" t_hand_tracking_sync *
t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib,
struct t_camera_extra_info extra_camera_info,
struct t_hand_tracking_create_info create_info,
const char *models_folder)
{
XRT_TRACE_MARKER();
@ -1095,6 +1117,9 @@ t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib,
hgt->views[0].hgt = hgt;
hgt->views[1].hgt = hgt; // :)
hgt->hand_masks_sink = create_info.masks_sink;
struct t_camera_extra_info &extra_camera_info = create_info.cams_info;
hgt->views[0].camera_info = extra_camera_info.views[0];
hgt->views[1].camera_info = extra_camera_info.views[1];

2
src/xrt/tracking/hand/mercury/hg_sync.hpp
View file

@ -18,6 +18,7 @@
#include "xrt/xrt_defines.h"
#include "xrt/xrt_frame.h"
#include "xrt/xrt_tracking.h"
#include "math/m_api.h"
#include "math/m_vec2.h"
@ -253,6 +254,7 @@ public:
struct u_sink_debug debug_sink_ann = {};
struct u_sink_debug debug_sink_model = {};
struct xrt_hand_masks_sink *hand_masks_sink;
float multiply_px_coord_for_undistort;