mirrors/monado
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/monado/monado.git synced 2024-12-28 18:46:18 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

t/slam: Add hand_masks sink

Browse source
This commit is contained in:
Mateo de Mayo 2024-01-23 13:21:49 -03:00 committed by Simon Zeni
parent 717336fa88
commit 4cfe58a5e9
2 changed files with 71 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
src/xrt/auxiliary/tracking/t_tracker_slam.cpp
View file

@ -264,6 +264,8 @@ struct TrackerSlam
struct xrt_frame_sink cam_sinks[XRT_TRACKING_MAX_SLAM_CAMS]; //!< Sends camera frames to the SLAM system
struct xrt_imu_sink imu_sink = {}; //!< Sends imu samples to the SLAM system
struct xrt_pose_sink gt_sink = {}; //!< Register groundtruth trajectory for stats
struct xrt_hand_masks_sink hand_masks_sink = {}; //!< Register latest masks to ignore
bool submit; //!< Whether to submit data pushed to sinks to the SLAM tracker
uint32_t cam_count; //!< Number of cameras used for tracking
@ -275,8 +277,10 @@ struct TrackerSlam
struct openvr_tracker *ovr_tracker; //!< OpenVR lighthouse tracker
// Used mainly for checking that the timestamps come in order
timepoint_ns last_imu_ts; //!< Last received IMU sample timestamp
vector<timepoint_ns> last_cam_ts; //!< Last received image timestamp per cam
timepoint_ns last_imu_ts; //!< Last received IMU sample timestamp
vector<timepoint_ns> last_cam_ts; //!< Last received image timestamp per cam
struct xrt_hand_masks_sample last_hand_masks; //!< Last received hand masks info
Mutex last_hand_masks_mutex; //!< Mutex for @ref last_hand_masks
// Prediction
@ -1273,6 +1277,17 @@ t_slam_gt_sink_push(struct xrt_pose_sink *sink, xrt_pose_sample *sample)
xrt_sink_push_pose(t.euroc_recorder->gt, sample);
}
//! Receive and register masks to use in the next image
extern "C" void
t_slam_hand_mask_sink_push(struct xrt_hand_masks_sink *sink, struct xrt_hand_masks_sample *hand_masks)
{
XRT_TRACE_MARKER();
auto &t = *container_of(sink, TrackerSlam, hand_masks_sink);
unique_lock lock(t.last_hand_masks_mutex);
t.last_hand_masks = *hand_masks;
}
//! Receive and send IMU samples to the external SLAM system
extern "C" void
t_slam_receive_imu(struct xrt_imu_sink *sink, struct xrt_imu_sample *s)
@ -1365,7 +1380,21 @@ receive_frame(TrackerSlam &t, struct xrt_frame *frame, uint32_t cam_index)
default: SLAM_ERROR("Unknown image format"); return;
}
// TODO masks
{
unique_lock lock(t.last_hand_masks_mutex);
for (auto &view : t.last_hand_masks.views) {
if (!view.enabled) {
continue;
}
for (auto &hand : view.hands) {
if (!hand.enabled) {
continue;
}
// TODO@mateosss: add_mask(hand.rect);
}
};
}
{
XRT_TRACE_IDENT(slam_push);
@ -1551,6 +1580,9 @@ t_slam_create(struct xrt_frame_context *xfctx,
t.gt_sink.push_pose = t_slam_gt_sink_push;
t.sinks.gt = &t.gt_sink;
t.hand_masks_sink.push_hand_masks = t_slam_hand_mask_sink_push;
t.sinks.hand_masks = &t.hand_masks_sink;
t.submit = config->submit_from_start;
t.cam_count = config->cam_count;
@ -1563,6 +1595,7 @@ t_slam_create(struct xrt_frame_context *xfctx,
t.last_imu_ts = INT64_MIN;
t.last_cam_ts = vector<timepoint_ns>(t.cam_count, INT64_MIN);
t.last_hand_masks = xrt_hand_masks_sample{};
t.pred_type = config->prediction;

35
src/xrt/include/xrt/xrt_tracking.h
View file

@ -139,6 +139,22 @@ struct xrt_pose_sample
struct xrt_pose pose;
};
/*!
* Masks (bounding boxes) of different hands from current views
*/
struct xrt_hand_masks_sample
{
struct xrt_hand_masks_sample_camera
{
bool enabled; //!< Whether any hand mask for this camera is being reported
struct xrt_hand_masks_sample_hand
{
bool enabled; //!< Whether a mask for this hand is being reported
struct xrt_rect rect; //!< The mask itself in pixel coordinates
} hands[2];
} views[XRT_TRACKING_MAX_SLAM_CAMS];
};
/*!
* @interface xrt_imu_sink
*
@ -167,6 +183,17 @@ struct xrt_pose_sink
void (*push_pose)(struct xrt_pose_sink *, struct xrt_pose_sample *sample);
};
/*!
* @interface xrt_hand_masks_sink
*
* An object to push @ref xrt_hand_masks_sample to.
*/
struct xrt_hand_masks_sink
{
void (*push_hand_masks)(struct xrt_hand_masks_sink *, struct xrt_hand_masks_sample *hand_masks);
};
/*!
* Container of pointers to sinks that could be used for a SLAM system. Sinks
* are considered disabled if they are null.
@ -177,6 +204,7 @@ struct xrt_slam_sinks
struct xrt_frame_sink *cams[XRT_TRACKING_MAX_SLAM_CAMS];
struct xrt_imu_sink *imu;
struct xrt_pose_sink *gt; //!< Can receive ground truth poses if available
struct xrt_hand_masks_sink *hand_masks;
};
/*!
@ -293,6 +321,13 @@ xrt_sink_push_pose(struct xrt_pose_sink *sink, struct xrt_pose_sample *sample)
sink->push_pose(sink, sample);
}
//! @public @memberof xrt_hand_masks_sink
static inline void
xrt_sink_push_hand_masks(struct xrt_hand_masks_sink *sink, struct xrt_hand_masks_sample *hand_masks)
{
sink->push_hand_masks(sink, hand_masks);
}
//! @public @memberof xrt_tracked_psmv
static inline void
xrt_tracked_psmv_get_tracked_pose(struct xrt_tracked_psmv *psmv,