d/ht: Add hand tracking driver code skeleton

CMakeLists.txt

@@ -163,6 +163,7 @@ cmake_dependent_option(XRT_BUILD_DRIVER_PSVR "Enable PSVR HMD driver" ON "HIDAPI
 cmake_dependent_option(XRT_BUILD_DRIVER_RS "Enable RealSense device driver" ON "realsense2_FOUND" OFF)
 cmake_dependent_option(XRT_BUILD_DRIVER_VIVE "Enable driver for HTC Vive, Vive Pro, Valve Index, and their controllers" ON "ZLIB_FOUND AND XRT_HAVE_LINUX" OFF)
 cmake_dependent_option(XRT_BUILD_DRIVER_OHMD "Enable OpenHMD driver" ON "OPENHMD_FOUND" OFF)
+cmake_dependent_option(XRT_BUILD_DRIVER_HANDTRACKING "Enable Camera Hand Tracking driver" ON "XRT_HAVE_V4L2" OFF)
 cmake_dependent_option(XRT_BUILD_DRIVER_DAYDREAM "Enable the Google Daydream View controller driver (BLE, via D-Bus)" ON "XRT_HAVE_DBUS" OFF)
 cmake_dependent_option(XRT_BUILD_DRIVER_ARDUINO "Enable Arduino input device with BLE via via D-Bus" ON "XRT_HAVE_DBUS" OFF)
 
@@ -182,7 +183,7 @@ cmake_dependent_option(XRT_BUILD_DRIVER_ANDROID "Enable Android sensors driver"
 
 # You can set this from a superproject to add a driver
 # All drivers must be listed in here to be included in the generated header!
-list(APPEND AVAILABLE_DRIVERS "ANDROID" ARDUINO DUMMY HDK HYDRA NS OHMD PSMV PSVR RS V4L2 VIVE DAYDREAM REMOTE SURVIVE)
+list(APPEND AVAILABLE_DRIVERS "ANDROID" ARDUINO DUMMY HDK HYDRA NS OHMD PSMV PSVR RS V4L2 VIVE DAYDREAM REMOTE SURVIVE HANDTRACKING)
 
 
 # Package name needs to be known by the native code itself.
@@ -290,18 +291,19 @@ message(STATUS "#    FEATURE_OPENXR_LAYER_EQUIRECT:        ${XRT_FEATURE_OPENXR_
 message(STATUS "#    FEATURE_OPENXR_LAYER_EQUIRECT_LEGACY: ${XRT_FEATURE_OPENXR_LAYER_EQUIRECT_LEGACY}")
 message(STATUS "#    FEATURE_STEAMVR_PLUGIN:               ${XRT_FEATURE_STEAMVR_PLUGIN}")
 message(STATUS "#")
-message(STATUS "#    DRIVER_ANDROID:  ${XRT_BUILD_DRIVER_ANDROID}")
+message(STATUS "#    DRIVER_ANDROID:      ${XRT_BUILD_DRIVER_ANDROID}")
-message(STATUS "#    DRIVER_ARDUINO:  ${XRT_BUILD_DRIVER_ARDUINO}")
+message(STATUS "#    DRIVER_ARDUINO:      ${XRT_BUILD_DRIVER_ARDUINO}")
-message(STATUS "#    DRIVER_DAYDREAM: ${XRT_BUILD_DRIVER_DAYDREAM}")
+message(STATUS "#    DRIVER_DAYDREAM:     ${XRT_BUILD_DRIVER_DAYDREAM}")
-message(STATUS "#    DRIVER_DUMMY:    ${XRT_BUILD_DRIVER_DUMMY}")
+message(STATUS "#    DRIVER_DUMMY:        ${XRT_BUILD_DRIVER_DUMMY}")
-message(STATUS "#    DRIVER_HDK:      ${XRT_BUILD_DRIVER_HDK}")
+message(STATUS "#    DRIVER_HDK:          ${XRT_BUILD_DRIVER_HDK}")
-message(STATUS "#    DRIVER_HYDRA:    ${XRT_BUILD_DRIVER_HYDRA}")
+message(STATUS "#    DRIVER_HYDRA:        ${XRT_BUILD_DRIVER_HYDRA}")
-message(STATUS "#    DRIVER_NS:       ${XRT_BUILD_DRIVER_NS}")
+message(STATUS "#    DRIVER_NS:           ${XRT_BUILD_DRIVER_NS}")
-message(STATUS "#    DRIVER_OHMD:     ${XRT_BUILD_DRIVER_OHMD}")
+message(STATUS "#    DRIVER_OHMD:         ${XRT_BUILD_DRIVER_OHMD}")
-message(STATUS "#    DRIVER_PSMV:     ${XRT_BUILD_DRIVER_PSMV}")
+message(STATUS "#    DRIVER_HANDTRACKING: ${XRT_BUILD_DRIVER_HANDTRACKING}")
-message(STATUS "#    DRIVER_PSVR:     ${XRT_BUILD_DRIVER_PSVR}")
+message(STATUS "#    DRIVER_PSMV:         ${XRT_BUILD_DRIVER_PSMV}")
-message(STATUS "#    DRIVER_RS:       ${XRT_BUILD_DRIVER_RS}")
+message(STATUS "#    DRIVER_PSVR:         ${XRT_BUILD_DRIVER_PSVR}")
-message(STATUS "#    DRIVER_REMOTE:   ${XRT_BUILD_DRIVER_REMOTE}")
+message(STATUS "#    DRIVER_RS:           ${XRT_BUILD_DRIVER_RS}")
-message(STATUS "#    DRIVER_SURVIVE:  ${XRT_BUILD_DRIVER_SURVIVE}")
+message(STATUS "#    DRIVER_REMOTE:       ${XRT_BUILD_DRIVER_REMOTE}")
-message(STATUS "#    DRIVER_VIVE:     ${XRT_BUILD_DRIVER_VIVE}")
+message(STATUS "#    DRIVER_SURVIVE:      ${XRT_BUILD_DRIVER_SURVIVE}")
+message(STATUS "#    DRIVER_VIVE:         ${XRT_BUILD_DRIVER_VIVE}")
 message(STATUS "#####----- Config -----#####")

meson.build

@@ -161,6 +161,12 @@ if openhmd.found() and ('auto' in drivers or 'ohmd' in drivers)
 	endif
 endif
 
+if has_v4l2_header and ('auto' in drivers or 'handtracking' in drivers)
+	if 'handtracking' not in drivers
+		drivers += ['handtracking']
+	endif
+endif
+
 if hidapi.found() and ('auto' in drivers or 'psvr' in drivers or 'hdk' in drivers)
 	if 'psvr' not in drivers
 		drivers += ['psvr']

meson_options.txt

@@ -3,7 +3,7 @@
 
 option('drivers',
 	type: 'array',
-	choices: ['auto', 'dummy', 'hdk', 'hydra', 'ns', 'ohmd', 'psmv', 'psvr', 'rs', 'v4l2', 'vive', 'survive', 'daydream', 'arduino', 'remote'],
+	choices: ['auto', 'dummy', 'hdk', 'hydra', 'ns', 'ohmd', 'psmv', 'psvr', 'rs', 'v4l2', 'vive', 'survive', 'daydream', 'arduino', 'remote', 'handtracking'],
 	value: ['auto'],
 	description: 'Set of drivers to build')
 

src/xrt/drivers/CMakeLists.txt

@@ -97,6 +97,18 @@ if(XRT_BUILD_DRIVER_OHMD)
 	list(APPEND ENABLED_HEADSET_DRIVERS openhmd)
 endif()
 
+if(XRT_BUILD_DRIVER_HANDTRACKING)
+	set(HT_SOURCE_FILES
+		ht/ht_driver.c
+		ht/ht_driver.h
+		ht/ht_interface.h
+		ht/ht_prober.c
+		)
+
+	add_library(drv_ht STATIC ${HT_SOURCE_FILES})
+	target_link_libraries(drv_ht PRIVATE xrt-interfaces aux_util aux_math)
+	list(APPEND ENABLED_DRIVERS ht)
+endif()
 
 if(XRT_BUILD_DRIVER_PSMV)
 	set(PSMOVE_SOURCE_FILES

src/xrt/drivers/ht/ht_driver.c

@@ -0,0 +1,114 @@
+// Copyright 2020, Collabora, Ltd.
+// SPDX-License-Identifier: BSL-1.0
+/*!
+ * @file
+ * @brief  Camera based hand tracking driver code.
+ * @author Christtoph Haag <christtoph.haag@collabora.com>
+ * @ingroup drv_ht
+ */
+
+#include "ht_driver.h"
+#include "util/u_device.h"
+#include "util/u_var.h"
+#include "util/u_debug.h"
+#include <string.h>
+
+struct ht_device
+{
+	struct xrt_device base;
+
+	enum u_logging_level ll;
+};
+
+DEBUG_GET_ONCE_LOG_OPTION(ht_log, "HT_LOG", U_LOGGING_WARN)
+
+#define HT_TRACE(htd, ...) U_LOG_XDEV_IFL_T(&htd->base, htd->ll, __VA_ARGS__)
+#define HT_DEBUG(htd, ...) U_LOG_XDEV_IFL_D(&htd->base, htd->ll, __VA_ARGS__)
+#define HT_INFO(htd, ...) U_LOG_XDEV_IFL_I(&htd->base, htd->ll, __VA_ARGS__)
+#define HT_WARN(htd, ...) U_LOG_XDEV_IFL_W(&htd->base, htd->ll, __VA_ARGS__)
+#define HT_ERROR(htd, ...) U_LOG_XDEV_IFL_E(&htd->base, htd->ll, __VA_ARGS__)
+
+static inline struct ht_device *
+ht_device(struct xrt_device *xdev)
+{
+	return (struct ht_device *)xdev;
+}
+
+static void
+ht_device_update_inputs(struct xrt_device *xdev)
+{
+	// Empty
+}
+
+static void
+ht_device_get_hand_tracking(struct xrt_device *xdev,
+                            enum xrt_input_name name,
+                            uint64_t at_timestamp_ns,
+                            struct xrt_hand_joint_set *out_value)
+{
+	struct ht_device *htd = ht_device(xdev);
+
+	enum xrt_hand hand;
+	int index;
+
+	if (name == XRT_INPUT_GENERIC_HAND_TRACKING_LEFT) {
+		HT_TRACE(htd, "Get left hand tracking data");
+		index = 0;
+		hand = XRT_HAND_LEFT;
+	} else if (name == XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT) {
+		HT_TRACE(htd, "Get right hand tracking data");
+		index = 1;
+		hand = XRT_HAND_RIGHT;
+	} else {
+		HT_ERROR(htd, "unknown input name for hand tracker");
+		return;
+	}
+
+}
+
+static void
+ht_device_destroy(struct xrt_device *xdev)
+{
+	struct ht_device *htd = ht_device(xdev);
+
+	// Remove the variable tracking.
+	u_var_remove_root(htd);
+
+	u_device_free(&htd->base);
+}
+
+struct xrt_device *
+ht_device_create(struct xrt_auto_prober *xap,
+                 cJSON *attached_data,
+                 struct xrt_prober *xp)
+{
+	enum u_device_alloc_flags flags =
+	    (enum u_device_alloc_flags)(U_DEVICE_ALLOC_TRACKING_NONE);
+
+	//! @todo 2 hands hardcoded
+	int num_hands = 2;
+
+	struct ht_device *htd =
+	    U_DEVICE_ALLOCATE(struct ht_device, flags, num_hands, 0);
+
+	htd->ll = debug_get_log_option_ht_log();
+
+	htd->base.update_inputs = ht_device_update_inputs;
+	htd->base.get_hand_tracking = ht_device_get_hand_tracking;
+	htd->base.destroy = ht_device_destroy;
+
+	strncpy(htd->base.str, "Camera based Hand Tracker",
+	        XRT_DEVICE_NAME_LEN);
+
+	htd->base.inputs[0].name = XRT_INPUT_GENERIC_HAND_TRACKING_LEFT;
+	htd->base.inputs[1].name = XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT;
+
+	htd->base.name = XRT_DEVICE_HAND_TRACKER;
+
+	u_var_add_root(htd, "Camera based Hand Tracker", true);
+	u_var_add_ro_text(htd, htd->base.str, "Name");
+
+	HT_DEBUG(htd, "Hand Tracker initialized!");
+
+	return &htd->base;
+}

src/xrt/drivers/ht/ht_driver.h

@@ -0,0 +1,26 @@
+// Copyright 2020, Collabora, Ltd.
+// SPDX-License-Identifier: BSL-1.0
+/*!
+ * @file
+ * @brief  Interface to camera based hand tracking driver code.
+ * @author Christtoph Haag <christtoph.haag@collabora.com>
+ * @ingroup drv_ht
+ */
+
+#pragma once
+
+#include "math/m_api.h"
+#include "xrt/xrt_device.h"
+#include "xrt/xrt_prober.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct xrt_device *
+ht_device_create();
+
+
+#ifdef __cplusplus
+}
+#endif

src/xrt/drivers/ht/ht_interface.h

@@ -0,0 +1,41 @@
+// Copyright 2029, Collabora, Ltd.
+// SPDX-License-Identifier: BSL-1.0
+/*!
+ * @file
+ * @brief  Interface to camera based hand tracking driver code.
+ * @author Christoph Haag <christoph.haag@collabora.com>
+ * @ingroup drv_ht
+ */
+
+#pragma once
+
+#include "math/m_api.h"
+#include "xrt/xrt_device.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*!
+ * @defgroup drv_ht Camera based hand tracking
+ * @ingroup drv
+ *
+ * @brief
+ */
+
+/*!
+ * Create a probe for camera based hand tracking.
+ *
+ * @ingroup drv_ht
+ */
+struct xrt_auto_prober *
+ht_create_auto_prober();
+
+/*!
+ * @dir drivers/handtracking
+ *
+ * @brief @ref drv_ht files.
+ */
+#ifdef __cplusplus
+}
+#endif

src/xrt/drivers/ht/ht_prober.c

@@ -0,0 +1,70 @@
+// Copyright 2020, Collabora, Ltd.
+// SPDX-License-Identifier: BSL-1.0
+/*!
+ * @file
+ * @brief  Camera based hand tracking prober code.
+ * @author Christoph Haag <christoph.haag@collabora.com>
+ * @ingroup drv_ht
+ */
+
+
+#include "xrt/xrt_prober.h"
+
+#include "util/u_misc.h"
+
+#include "ht_interface.h"
+#include "ht_driver.h"
+
+/*!
+ * @implements xrt_auto_prober
+ */
+struct ht_prober
+{
+	struct xrt_auto_prober base;
+};
+
+//! @private @memberof ht_prober
+static inline struct ht_prober *
+ht_prober(struct xrt_auto_prober *p)
+{
+	return (struct ht_prober *)p;
+}
+
+//! @public @memberof ht_prober
+static void
+ht_prober_destroy(struct xrt_auto_prober *p)
+{
+	struct ht_prober *htp = ht_prober(p);
+
+	free(htp);
+}
+
+//! @public @memberof ht_prober
+static struct xrt_device *
+ht_prober_autoprobe(struct xrt_auto_prober *xap,
+                    cJSON *attached_data,
+                    bool no_hmds,
+                    struct xrt_prober *xp)
+{
+	struct ht_prober *htp = ht_prober(xap);
+
+	struct xrt_device *xdev = ht_device_create(xap, attached_data, xp);
+
+	xdev->orientation_tracking_supported = true;
+	xdev->position_tracking_supported = true;
+	xdev->hand_tracking_supported = true;
+	xdev->device_type = XRT_DEVICE_TYPE_HAND_TRACKER;
+
+	return xdev;
+}
+
+struct xrt_auto_prober *
+ht_create_auto_prober()
+{
+	struct ht_prober *htp = U_TYPED_CALLOC(struct ht_prober);
+	htp->base.name = "Camera Hand Tracking";
+	htp->base.destroy = ht_prober_destroy;
+	htp->base.lelo_dallas_autoprobe = ht_prober_autoprobe;
+
+	return &htp->base;
+}

src/xrt/drivers/meson.build

@@ -58,6 +58,19 @@ lib_drv_ns = static_library(
 	build_by_default: 'ns' in drivers,
 )
 
+lib_drv_ht = static_library(
+	'drv_ht',
+	files(
+		'ht/ht_driver.c',
+		'ht/ht_driver.h',
+		'ht/ht_interface.h',
+		'ht/ht_prober.c',
+	),
+	include_directories: xrt_include,
+	dependencies: [aux],
+	build_by_default: 'handtracking' in drivers,
+)
+
 lib_drv_ohmd = static_library(
 	'drv_ohmd',
 	files(

src/xrt/include/xrt/xrt_defines.h

@@ -418,7 +418,9 @@ enum xrt_device_name
 	XRT_DEVICE_INDEX_CONTROLLER,
 	XRT_DEVICE_VIVE_WAND,
 	XRT_DEVICE_VIVE_TRACKER_GEN1,
-	XRT_DEVICE_VIVE_TRACKER_GEN2
+	XRT_DEVICE_VIVE_TRACKER_GEN2,
+
+	XRT_DEVICE_HAND_TRACKER,
 };
 
 /*!
@@ -433,7 +435,8 @@ enum xrt_device_type
 	XRT_DEVICE_TYPE_RIGHT_HAND_CONTROLLER,
 	XRT_DEVICE_TYPE_LEFT_HAND_CONTROLLER,
 	XRT_DEVICE_TYPE_ANY_HAND_CONTROLLER,
-	XRT_DEVICE_TYPE_GENERIC_TRACKER
+	XRT_DEVICE_TYPE_GENERIC_TRACKER,
+	XRT_DEVICE_TYPE_HAND_TRACKER,
 };
 
 /*!
@@ -497,7 +500,7 @@ enum xrt_input_name
 	XRT_INPUT_GENERIC_HEAD_POSE                  = XRT_INPUT_NAME(0x0000, POSE),
 	XRT_INPUT_GENERIC_HEAD_DETECT                = XRT_INPUT_NAME(0x0001, BOOLEAN),
 	XRT_INPUT_GENERIC_HAND_TRACKING_LEFT         = XRT_INPUT_NAME(0x0002, HAND_TRACKING),
-	XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT        = XRT_INPUT_NAME(0x0003, HAND_TRACKING),
+	XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT        = XRT_INPUT_NAME(0x0004, HAND_TRACKING),
 
 	XRT_INPUT_SIMPLE_SELECT_CLICK                = XRT_INPUT_NAME(0x0010, BOOLEAN),
 	XRT_INPUT_SIMPLE_MENU_CLICK                  = XRT_INPUT_NAME(0x0011, BOOLEAN),

src/xrt/targets/cli/cli_cmd_probe.c

@@ -52,6 +52,10 @@ cli_cmd_probe(int argc, const char **argv)
 	printf(" OpenHMD,");
 #endif
 
+#ifdef XRT_BUILD_DRIVER_HANDTRACKING
+	printf(" Hand Tracking,");
+#endif
+
 #ifdef XRT_BUILD_DRIVER_DAYDREAM
 	printf(" Daydream,");
 #endif

src/xrt/targets/common/CMakeLists.txt

@@ -48,6 +48,10 @@ if(XRT_BUILD_DRIVER_OHMD)
 	target_link_libraries(target_lists PRIVATE drv_ohmd)
 endif()
 
+if(XRT_BUILD_DRIVER_HANDTRACKING)
+	target_link_libraries(target_lists PRIVATE drv_ht)
+endif()
+
 if(XRT_BUILD_DRIVER_PSMV)
 	target_link_libraries(target_lists PRIVATE drv_psmv)
 endif()

src/xrt/targets/common/target_lists.c

@@ -26,6 +26,10 @@
 #include "ohmd/oh_interface.h"
 #endif
 
+#ifdef XRT_BUILD_DRIVER_HANDTRACKING
+#include "ht/ht_interface.h"
+#endif
+
 #ifdef XRT_BUILD_DRIVER_NS
 #include "north_star/ns_interface.h"
 #endif
@@ -133,6 +137,10 @@ xrt_auto_prober_creator target_auto_list[] = {
     oh_create_auto_prober,
 #endif
 
+#ifdef XRT_BUILD_DRIVER_HANDTRACKING
+    ht_create_auto_prober,
+#endif
+
 #ifdef XRT_BUILD_DRIVER_NS
     // North star driver here for now.
     ns_create_auto_prober,

src/xrt/targets/meson.build

@@ -32,6 +32,10 @@ if 'ohmd' in drivers
 	driver_libs += [lib_drv_ohmd]
 endif
 
+if 'handtracking' in drivers
+	driver_libs += [lib_drv_ht]
+endif
+
 if 'psmv' in drivers
 	driver_libs += [lib_drv_psmv]
 endif