diff --git a/src/xrt/drivers/ht/ht_driver.c b/src/xrt/drivers/ht/ht_driver.c index cffa9e9fa..423e145a3 100644 --- a/src/xrt/drivers/ht/ht_driver.c +++ b/src/xrt/drivers/ht/ht_driver.c @@ -241,7 +241,7 @@ int ht_device_create(struct xrt_frame_context *xfctx, struct t_stereo_camera_calibration *calib, enum t_hand_tracking_algorithm algorithm_choice, - struct t_image_boundary_info boundary_info, + struct t_camera_extra_info extra_camera_info, struct xrt_slam_sinks **out_sinks, struct xrt_device **out_device) { @@ -253,7 +253,7 @@ ht_device_create(struct xrt_frame_context *xfctx, switch (algorithm_choice) { case HT_ALGORITHM_MERCURY: { - sync = t_hand_tracking_sync_mercury_create(calib, boundary_info); + sync = t_hand_tracking_sync_mercury_create(calib, extra_camera_info); } break; case HT_ALGORITHM_OLD_RGB: { //!@todo Either have this deal with the output space correctly, or have everything use LEFT_CAMERA @@ -291,9 +291,9 @@ ht_device_create_depthai_ov9282() struct t_hand_tracking_sync *sync; - struct t_image_boundary_info info; - info.views[0].type = HT_IMAGE_BOUNDARY_NONE; - info.views[1].type = HT_IMAGE_BOUNDARY_NONE; + struct t_camera_extra_info info; + info.views[0].boundary_type = HT_IMAGE_BOUNDARY_NONE; + info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE; sync = t_hand_tracking_sync_mercury_create(calib, info); diff --git a/src/xrt/drivers/ht/ht_interface.h b/src/xrt/drivers/ht/ht_interface.h index f406311bc..8ab5532d3 100644 --- a/src/xrt/drivers/ht/ht_interface.h +++ b/src/xrt/drivers/ht/ht_interface.h @@ -55,7 +55,7 @@ int ht_device_create(struct xrt_frame_context *xfctx, struct t_stereo_camera_calibration *calib, enum t_hand_tracking_algorithm algorithm_choice, - struct t_image_boundary_info boundary_info, + struct t_camera_extra_info extra_camera_info, struct xrt_slam_sinks **out_sinks, struct xrt_device **out_device); diff --git a/src/xrt/drivers/wmr/wmr_hmd.c b/src/xrt/drivers/wmr/wmr_hmd.c index a54bff6b9..19f9b6afc 100644 --- a/src/xrt/drivers/wmr/wmr_hmd.c +++ b/src/xrt/drivers/wmr/wmr_hmd.c @@ -1493,14 +1493,14 @@ wmr_hmd_hand_track(struct wmr_hmd *wh, #ifdef XRT_BUILD_DRIVER_HANDTRACKING //!@todo Turning it off is okay for now, but we should plug metric_radius (or whatever it's called) in, at some //! point. - struct t_image_boundary_info boundary_info; - boundary_info.views[0].type = HT_IMAGE_BOUNDARY_NONE; - boundary_info.views[1].type = HT_IMAGE_BOUNDARY_NONE; + struct t_camera_extra_info extra_camera_info; + extra_camera_info.views[0].boundary_type = HT_IMAGE_BOUNDARY_NONE; + extra_camera_info.views[1].boundary_type = HT_IMAGE_BOUNDARY_NONE; int create_status = ht_device_create(&wh->tracking.xfctx, // stereo_calib, // HT_ALGORITHM_MERCURY, // - boundary_info, // + extra_camera_info, // &sinks, // &device); if (create_status != 0) { diff --git a/src/xrt/include/tracking/t_hand_tracking.h b/src/xrt/include/tracking/t_hand_tracking.h index 77661eecd..1373cfcb8 100644 --- a/src/xrt/include/tracking/t_hand_tracking.h +++ b/src/xrt/include/tracking/t_hand_tracking.h @@ -23,6 +23,9 @@ extern "C" { * * Currently used by hand-tracking to determine if parts of the hand are not visible to the camera, ie. they are outside * of the camera's vignette. + * + * Feel free to move this out of t_hand_tracking if this becomes more generally applicable. + * * @ingroup xrt_iface */ enum t_image_boundary_type @@ -36,6 +39,9 @@ enum t_image_boundary_type * * Currently used by hand-tracking to determine if parts of the hand are not visible to the camera, ie. they are outside * of the camera's vignette. + * + * Feel free to move this out of t_hand_tracking if this becomes more generally applicable. + * * @ingroup xrt_iface */ struct t_image_boundary_circle @@ -49,31 +55,53 @@ struct t_image_boundary_circle }; /*! - * @brief Image boundary for one view. + * @brief Logical orientation of the camera image, relative to the user's head. + * For example, Rift S uses CAMERA_ORIENTATION_90 for the two front cameras. + * + * Feel free to move this out of t_hand_tracking if this becomes more generally applicable. + * + */ +enum t_camera_orientation +{ + CAMERA_ORIENTATION_0 = 0, // Normal "horizontal" orientation + CAMERA_ORIENTATION_90 = 90, // Camera rotated 90° to the right + CAMERA_ORIENTATION_180 = 180, // Camera rotated 180° upside down + CAMERA_ORIENTATION_270 = 270, // Camera rotated 270° to the left +}; + + +/*! + * @brief Information about image boundary and camera orientation for one view. + * + * Currently used by hand-tracking to determine if parts of the hand are not + * visible to the camera, ie. they are outside of the camera's vignette. * - * Currently used by hand-tracking to determine if parts of the hand are not visible to the camera, ie. they are outside - * of the camera's vignette. * @ingroup xrt_iface */ -struct t_image_boundary_info_one_view +struct t_camera_extra_info_one_view { - enum t_image_boundary_type type; + enum t_image_boundary_type boundary_type; + union { struct t_image_boundary_circle circle; } boundary; + + enum t_camera_orientation camera_orientation; }; /*! - * @brief Image boundaries for all the cameras used in a tracking system. + * @brief Information about image boundaries and camera orientations for all the + * cameras used in a tracking system. + * + * Currently used by hand-tracking to determine if parts of the hand are not + * visible to the camera, ie. they are outside of the camera's vignette. * - * Currently used by hand-tracking to determine if parts of the hand are not visible to the camera, ie. they are outside - * of the camera's vignette. * @ingroup xrt_iface */ -struct t_image_boundary_info +struct t_camera_extra_info { //!@todo Hardcoded to 2 - needs to increase as we support headsets with more cameras. - struct t_image_boundary_info_one_view views[2]; + struct t_camera_extra_info_one_view views[2]; }; /*! diff --git a/src/xrt/targets/common/target_builder_lighthouse.c b/src/xrt/targets/common/target_builder_lighthouse.c index 0f70795bf..3e86e7d53 100644 --- a/src/xrt/targets/common/target_builder_lighthouse.c +++ b/src/xrt/targets/common/target_builder_lighthouse.c @@ -188,9 +188,9 @@ lighthouse_hand_track(struct u_system_devices *usysd, LH_ASSERT_(stereo_calib != NULL); - struct t_image_boundary_info info; - info.views[0].type = HT_IMAGE_BOUNDARY_CIRCLE; - info.views[1].type = HT_IMAGE_BOUNDARY_CIRCLE; + struct t_camera_extra_info info; + info.views[0].boundary_type = HT_IMAGE_BOUNDARY_CIRCLE; + info.views[1].boundary_type = HT_IMAGE_BOUNDARY_CIRCLE; //!@todo This changes by like 50ish pixels from device to device. For now, the solution is simple: just diff --git a/src/xrt/tracking/hand/mercury/hg_interface.h b/src/xrt/tracking/hand/mercury/hg_interface.h index 4592fb61c..a8a4593bf 100644 --- a/src/xrt/tracking/hand/mercury/hg_interface.h +++ b/src/xrt/tracking/hand/mercury/hg_interface.h @@ -23,7 +23,7 @@ extern "C" { */ struct t_hand_tracking_sync * t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib, - struct t_image_boundary_info boundary_info); + struct t_camera_extra_info extra_camera_info); #ifdef __cplusplus } // extern "C" diff --git a/src/xrt/tracking/hand/mercury/hg_model.cpp b/src/xrt/tracking/hand/mercury/hg_model.cpp index b42b1bc5e..f3befa279 100644 --- a/src/xrt/tracking/hand/mercury/hg_model.cpp +++ b/src/xrt/tracking/hand/mercury/hg_model.cpp @@ -29,63 +29,93 @@ namespace xrt::tracking::hand::mercury { } \ } while (0) - static cv::Matx23f -blackbar(const cv::Mat &in, cv::Mat &out, xrt_size out_size) +blackbar(const cv::Mat &in, enum t_camera_orientation rot, cv::Mat &out, xrt_size out_size) { -#if 1 // Easy to think about, always right, but pretty slow: // Get a matrix from the original to the scaled down / blackbar'd image, then get one that goes back. // Then just warpAffine() it. // Easy in programmer time - never have to worry about off by one, special cases. We can come back and optimize // later. + bool swapped_wh = false; + float in_w, in_h; - // Do the black bars need to be on top and bottom, or on left and right? - float scale_down_w = (float)out_size.w / (float)in.cols; // 128/1280 = 0.1 - float scale_down_h = (float)out_size.h / (float)in.rows; // 128/800 = 0.16 + switch (rot) { + case CAMERA_ORIENTATION_90: + case CAMERA_ORIENTATION_270: + // Swap width and height + in_w = in.rows; + in_h = in.cols; + swapped_wh = true; + break; + default: + in_w = in.cols; + in_h = in.rows; + break; + } + + // Figure out from the rotation and frame sizes if the black bars need to be on top and bottom, or on left and + // right? + float scale_down_w = (float)out_size.w / in_w; // 128/1280 = 0.1 + float scale_down_h = (float)out_size.h / in_h; // 128/800 = 0.16 float scale_down = fmin(scale_down_w, scale_down_h); // 0.1 - float width_inside = (float)in.cols * scale_down; - float height_inside = (float)in.rows * scale_down; + float width_inside, height_inside; + + if (swapped_wh) { + width_inside = (float)in.rows * scale_down; + height_inside = (float)in.cols * scale_down; + } else { + width_inside = (float)in.cols * scale_down; + height_inside = (float)in.rows * scale_down; + } float translate_x = (out_size.w - width_inside) / 2; // should be 0 for 1280x800 float translate_y = (out_size.h - height_inside) / 2; // should be (1280-800)/2 = 240 cv::Matx23f go; - // clang-format off - go(0,0) = scale_down; go(0,1) = 0.0f; go(0,2) = translate_x; - go(1,0) = 0.0f; go(1,1) = scale_down; go(1,2) = translate_y; - // clang-format on + cv::Point2f center(in.rows / 2, in.cols / 2); + + switch (rot) { + case CAMERA_ORIENTATION_0: + // clang-format off + go(0,0) = scale_down; go(0,1) = 0.0f; go(0,2) = translate_x; + go(1,0) = 0.0f; go(1,1) = scale_down; go(1,2) = translate_y; + // clang-format on + break; + case CAMERA_ORIENTATION_90: + // clang-format off + go(0,0) = 0.0f; go(0,1) = scale_down; go(0,2) = translate_x; + go(1,0) = -scale_down; go(1,1) = 0.0f; go(1,2) = translate_y+out_size.h-1; + // clang-format on + break; + case CAMERA_ORIENTATION_180: + // clang-format off + go(0,0) = -scale_down; go(0,1) = 0.0f; go(0,2) = translate_x+out_size.w-1; + go(1,0) = 0.0f; go(1,1) = -scale_down; go(1,2) = translate_y+out_size.h-1; + // clang-format on + break; + case CAMERA_ORIENTATION_270: + // clang-format off + go(0,0) = 0.0f; go(0,1) = -scale_down; go(0,2) = translate_x+out_size.w-1; + go(1,0) = scale_down; go(1,1) = 0.0f; go(1,2) = translate_y; + // clang-format on + break; + } cv::warpAffine(in, out, go, cv::Size(out_size.w, out_size.h)); - cv::Matx23f ret; + // Return the inverse affine transform by passing + // through a 3x3 rotation matrix + cv::Mat e = cv::Mat::eye(3, 3, CV_32F); + cv::Mat tmp = e(cv::Rect(0, 0, 3, 2)); + cv::Mat(go).copyTo(tmp); - // clang-format off - ret(0,0) = 1.0f/scale_down; ret(0,1) = 0.0f; ret(0,2) = -translate_x/scale_down; - ret(1,0) = 0.0f; ret(1,1) = 1.0f/scale_down; ret(1,2) = -translate_y/scale_down; - // clang-format on + e = e.inv(); + cv::Matx23f ret = e(cv::Rect(0, 0, 3, 2)); return ret; -#else - // Fast, always wrong if the input isn't square. You'd end up using something like this, plus some - // copyMakeBorder if you want to optimize. - if (aspect_ratio_input == aspect_ratio_output) { - cv::resize(in, out, {out_size.w, out_size.h}); - cv::Matx23f ret; - float scale_from_out_to_in = (float)in.cols / (float)out_size.w; - // clang-format off - ret(0,0) = scale_from_out_to_in; ret(0,1) = 0.0f; ret(0,2) = 0.0f; - ret(1,0) = 0.0f; ret(1,1) = scale_from_out_to_in; ret(1,2) = 0.0f; - // clang-format on - cv::imshow("hi", out); - cv::waitKey(1); - return ret; - } - assert(!"Uh oh! Unimplemented!"); - return {}; -#endif } static inline int @@ -275,7 +305,8 @@ run_hand_detection(void *ptr) desire.h = 240; desire.w = 320; - cv::Matx23f go_back = blackbar(data_400x640, _240x320_uint8, desire); + cv::Matx23f go_back = blackbar(data_400x640, view->camera_info.camera_orientation, _240x320_uint8, desire); + cv::Mat _240x320(cv::Size(320, 240), CV_32FC1, wrap->data, 320 * sizeof(float)); diff --git a/src/xrt/tracking/hand/mercury/hg_sync.cpp b/src/xrt/tracking/hand/mercury/hg_sync.cpp index 019707ee0..58edd0eb9 100644 --- a/src/xrt/tracking/hand/mercury/hg_sync.cpp +++ b/src/xrt/tracking/hand/mercury/hg_sync.cpp @@ -165,7 +165,7 @@ getModelsFolder(struct HandTracking *hgt) template <typename Vec> static inline bool -check_outside_view(struct HandTracking *hgt, struct t_image_boundary_info_one_view boundary, Vec &keypoint) +check_outside_view(struct HandTracking *hgt, struct t_camera_extra_info_one_view boundary, Vec &keypoint) { // Regular case - the keypoint is literally outside the image if (keypoint.y > hgt->calibration_one_view_size_px.h || // @@ -175,7 +175,7 @@ check_outside_view(struct HandTracking *hgt, struct t_image_boundary_info_one_vi return true; } - switch (boundary.type) { + switch (boundary.boundary_type) { // No boundary, and we passed the previous check. Not outside the view. case HT_IMAGE_BOUNDARY_NONE: return false; break; case HT_IMAGE_BOUNDARY_CIRCLE: { @@ -254,7 +254,7 @@ back_project(struct HandTracking *hgt, xrt_vec2 keypoints_global[26]; bool outside_view[26] = {}; for (int i = 0; i < 26; i++) { - if (check_outside_view(hgt, hgt->image_boundary_info.views[view_idx], out[i]) || + if (check_outside_view(hgt, hgt->views[view_idx].camera_info, out[i]) || any_joint_behind_camera) { outside_view[i] = true; if (num_outside != NULL) { @@ -568,9 +568,9 @@ scribble_image_boundary(struct HandTracking *hgt) struct ht_view *view = &hgt->views[view_idx]; cv::Mat &debug_frame = view->debug_out_to_this; - t_image_boundary_info_one_view &info = hgt->image_boundary_info.views[view_idx]; + t_camera_extra_info_one_view &info = hgt->views[view_idx].camera_info; - if (info.type == HT_IMAGE_BOUNDARY_CIRCLE) { + if (info.boundary_type == HT_IMAGE_BOUNDARY_CIRCLE) { int center_x = hgt->last_frame_one_view_size_px.w * info.boundary.circle.normalized_center.x; int center_y = hgt->last_frame_one_view_size_px.h * info.boundary.circle.normalized_center.y; cv::circle(debug_frame, {center_x, center_y}, @@ -913,7 +913,7 @@ using namespace xrt::tracking::hand::mercury; extern "C" t_hand_tracking_sync * t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib, - struct t_image_boundary_info boundary_info) + struct t_camera_extra_info extra_camera_info) { XRT_TRACE_MARKER(); @@ -964,7 +964,8 @@ t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib, hgt->views[0].hgt = hgt; hgt->views[1].hgt = hgt; // :) - hgt->image_boundary_info = boundary_info; + hgt->views[0].camera_info = extra_camera_info.views[0]; + hgt->views[1].camera_info = extra_camera_info.views[1]; init_hand_detection(hgt, &hgt->views[0].detection); init_hand_detection(hgt, &hgt->views[1].detection); diff --git a/src/xrt/tracking/hand/mercury/hg_sync.hpp b/src/xrt/tracking/hand/mercury/hg_sync.hpp index ca57c4ff8..abcbce692 100644 --- a/src/xrt/tracking/hand/mercury/hg_sync.hpp +++ b/src/xrt/tracking/hand/mercury/hg_sync.hpp @@ -133,6 +133,8 @@ struct ht_view onnx_wrap keypoint[2]; int view; + struct t_camera_extra_info_one_view camera_info; + cv::Mat distortion; cv::Matx<double, 3, 3> cameraMatrix; cv::Matx33d rotate_camera_to_stereo_camera; // R1 or R2 @@ -242,7 +244,6 @@ public: struct xrt_pose left_in_right = {}; - struct t_image_boundary_info image_boundary_info; u_frame_times_widget ft_widget = {};