diff --git a/.editorconfig b/.editorconfig index 3ff654b59..178f43667 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,8 +1,8 @@ -# To use this config on you editor, follow the instructions at: +# To use this config on your editor, follow the instructions at: # http://editorconfig.org # SPDX-License-Identifier: CC0-1.0 -# SPDX-FileCopyrightText: 2018-2020 Collabora, Ltd. and the Monado contributors +# SPDX-FileCopyrightText: 2018-2021 Collabora, Ltd. and the Monado contributors root = true @@ -13,7 +13,7 @@ insert_final_newline = true [*.{c,h,cpp}] indent_style = tab indent_size = 8 -max_line_length = 80 +max_line_length = 120 [*.py] indent_size = 4 diff --git a/.gitignore b/.gitignore index d3b8d96dc..d2c75393f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,19 +9,34 @@ CMakeLists.txt.user.* *Makefile *cmake_install.cmake *libopenxr_monado.so +*.a CMakeDoxyfile.in CMakeDoxygenDefaults.cmake +CTestTestfile.cmake +DartConfiguration.tcl doc/Doxyfile doc/html/ doc/latex/ openxr_monado-dev.json openxr_monado.json +src/xrt/auxiliary/bindings/b_generated_bindings.* +src/xrt/auxiliary/u_git_tag.c src/xrt/compositor/shaders/*.vert.h src/xrt/compositor/shaders/*.frag.h +src/xrt/compositor/shaders/*.comp.h +src/xrt/include/xrt/xrt_config_*.h +src/xrt/ipc/*_generated.* src/xrt/targets/cli/monado-cli +src/xrt/targets/ctl/monado-ctl src/xrt/targets/gui/monado-gui +src/xrt/targets/service/monado-service +src/xrt/targets/openxr/active_runtime.cmake src/xrt/targets/openxr/intermediate_manifest.json +src/xrt/targets/openxr/make_manifest.cmake src/xrt/targets/targets_enabled_drivers.h +steamvr-monado/ +tests/tests_generic_callbacks +tests/tests_input_transform # Ignore Python caches __pycache__/ @@ -71,7 +86,16 @@ local.properties .cxx .settings .project +.classpath gradlew gradlew.bat gradle-wrapper.jar + +# Ignore Sourcetrail things +*.srctrlbm +*.srctrldb +*.srctrlprj + +# Ignore clangd things +.cache/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8a3f24cbc..bc27de217 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,32 +1,46 @@ # SPDX-License-Identifier: CC0-1.0 -# SPDX-FileCopyrightText: 2018-2020 Collabora, Ltd. and the Monado contributors +# SPDX-FileCopyrightText: 2018-2021 Collabora, Ltd. and the Monado contributors variables: FDO_UPSTREAM_REPO: monado/monado -.templates_sha: &templates_sha 322bf2b8f29b6491caeb13861201e96969ddc169 - -.package_only_branch: &package_only_branch master +.templates_sha: &templates_sha db8eb22cd1abb036560faaebd36a38565a3ebda2 # Variables listing packages for Debian-based distros .monado.variables.debian-based-packages: variables: - CORE_REQUIRED_PACKAGES: "build-essential git wget unzip cmake meson ninja-build libeigen3-dev curl patch python3 pkg-config libx11-dev libx11-xcb-dev libxxf86vm-dev libxrandr-dev libxcb-randr0-dev libvulkan-dev glslang-tools libglvnd-dev libgl1-mesa-dev ca-certificates libusb-1.0-0-dev libudev-dev" - FEATURE_PACKAGES: "libhidapi-dev libwayland-dev libuvc-dev libavcodec-dev libopencv-dev libv4l-dev libcjson-dev libsdl2-dev libegl1-mesa-dev libdbus-1-dev" - PACKAGING_PACKAGES: "devscripts debhelper osc osc-plugins-dput dput-ng gettext-base markdown" + # Packages required for build and some other basic jobs + CORE_REQUIRED_PACKAGES: "build-essential git wget unzip cmake meson ninja-build libeigen3-dev curl patch python3 pkg-config libx11-dev libx11-xcb-dev libxxf86vm-dev libxrandr-dev libxcb-randr0-dev libvulkan-dev glslang-tools libglvnd-dev libgl1-mesa-dev ca-certificates libusb-1.0-0-dev libudev-dev" + + # These are optional packages, that we're building against to ensure we build as much code as possible + FEATURE_PACKAGES: "libhidapi-dev libwayland-dev libuvc-dev libavcodec-dev libopencv-dev libv4l-dev libcjson-dev libsdl2-dev libegl1-mesa-dev libdbus-1-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libsystemd-dev libbsd-dev" + + # Only used for building packages + PACKAGING_PACKAGES: "devscripts debhelper dput-ng gettext-base markdown doxygen graphviz" + + # Used for ancillary "not compilation" jobs/features, like docs, changelogs, formatting, etc. TOOLS_REQUIRED_PACKAGES: "clang-format-7 codespell doxygen graphviz python3-pip python3-click" + # The NDK builder uses only these packages + NDK_PACKAGES: "git wget unzip cmake meson ninja-build libeigen3-dev python3 pkg-config ca-certificates glslang-tools" + # Variables for build and usage of Debian 10 (Buster) image .monado.variables.debian:buster: variables: FDO_DISTRIBUTION_VERSION: buster - FDO_DISTRIBUTION_TAG: "2020-07-10.0" + FDO_DISTRIBUTION_TAG: "2021-04-15.0" # Variables for build and usage of Ubuntu 20.04 LTS (Focal) image .monado.variables.ubuntu:focal: variables: FDO_DISTRIBUTION_VERSION: "20.04" - FDO_DISTRIBUTION_TAG: "2020-07-10.0" + FDO_DISTRIBUTION_TAG: "2021-04-15.0" + +# Variables for build and usage of Ubuntu 20.10 (Groovy) image +.monado.variables.ubuntu:groovy: + variables: + FDO_DISTRIBUTION_VERSION: "20.10" + FDO_DISTRIBUTION_TAG: "2021-04-15.0" # Variables for build and usage of Debian 10 (Buster) + Android NDK image .monado.variables.debian:buster-ndk: @@ -38,7 +52,7 @@ variables: # Variables for build and usage of Arch Linux image .monado.variables.arch:rolling: variables: - FDO_DISTRIBUTION_TAG: "2020-05-12.0" + FDO_DISTRIBUTION_TAG: "2021-09-14.0" include: - project: "freedesktop/ci-templates" @@ -95,7 +109,7 @@ arch:container_prep: FDO_DISTRIBUTION_PACKAGES: "git gcc clang cmake meson ninja pkgconfig python3 diffutils patch doxygen graphviz eigen hidapi libxrandr mesa glslang vulkan-headers vulkan-icd-loader check glfw-x11 libusb opencv gtk3 ffmpeg v4l-utils qt5-base" # Ubuntu Focal (x64) -ubuntu:container_prep: +ubuntu:focal:container_prep: stage: container_prep extends: - .monado.variables.ubuntu:focal @@ -106,6 +120,18 @@ ubuntu:container_prep: # a list of packages to install - assembled from .monado.variables.debian-based-packages FDO_DISTRIBUTION_PACKAGES: "${CORE_REQUIRED_PACKAGES} ${FEATURE_PACKAGES} ${PACKAGING_PACKAGES} ${TOOLS_REQUIRED_PACKAGES}" +# Ubuntu Groovy (x64) +ubuntu:groovy:container_prep: + stage: container_prep + extends: + - .monado.variables.ubuntu:groovy + - .monado.variables.container-prep-base + - .monado.variables.debian-based-packages + - .fdo.container-build@ubuntu # from ci-templates + variables: + # a list of packages to install - assembled from .monado.variables.debian-based-packages + FDO_DISTRIBUTION_PACKAGES: "${CORE_REQUIRED_PACKAGES} ${FEATURE_PACKAGES} ${PACKAGING_PACKAGES}" + # Debian Buster + the Android NDK in /opt/android-ndk # The NDK itself gets installed by .gitlab-ci/ndk:container_prep.sh ndk:container_prep: @@ -117,7 +143,7 @@ ndk:container_prep: variables: # Repo suffix is set in .monado.variables.debian:buster-ndk # a list of packages to install - FDO_DISTRIBUTION_PACKAGES: "git wget unzip cmake meson ninja-build libeigen3-dev python3 pkg-config ca-certificates glslang-tools" + FDO_DISTRIBUTION_PACKAGES: "${NDK_PACKAGES}" # Style check job format-and-spellcheck: @@ -133,10 +159,20 @@ format-and-spellcheck: expire_in: 1 week when: on_failure +# Verify REUSE compliance +reuse: + stage: build + image: + name: fsfe/reuse:latest + entrypoint: [""] + script: + - reuse lint + # "Base" job for a CMake build .monado.base-job.build-cmake: stage: build script: + - rm -rf build - mkdir build - pushd build - cmake -GNinja .. $CMAKE_ARGS @@ -149,6 +185,7 @@ format-and-spellcheck: .monado.base-job.build-meson: stage: build script: + - rm -rf build - mkdir build - pushd build - meson .. $MESON_ARGS @@ -260,11 +297,46 @@ debian:cmake:32bit: # OpenCV and local OpenHMD doesn't play nicely with us in multi-arch. CMAKE_ARGS: -DCMAKE_TOOLCHAIN_FILE=../.gitlab-ci/i386.cmake -DXRT_HAVE_OPENCV=off -DXRT_BUILD_DRIVER_OHMD=off +# Base of Android NDK builds. +# Takes the last :-delimited part of the name as the ABI to build for, +# so you don't need to do anything other than "extends" in the job +.monado.ndk:build-base: + stage: build + extends: + - .monado.variables.debian:buster-ndk + - .fdo.suffixed-image@debian # from ci-templates + variables: + ANDROID_PLATFORM: 26 + script: + - mkdir build + - pushd build + # This extracts the ABI from the job name + - export ABI=$(echo $CI_JOB_NAME | cut --delimiter=":" -f 2) + # Note we are pointing CMake to the host install of Eigen3 because it's header-only + # and thus this is safe to do. + - cmake -GNinja .. -DANDROID_PLATFORM=$ANDROID_PLATFORM -DANDROID_ABI=$ABI -DCMAKE_TOOLCHAIN_FILE=/opt/android-ndk/build/cmake/android.toolchain.cmake -DEigen3_DIR=/usr/lib/cmake/eigen3/ + - grep "^XRT_" CMakeCache.txt + - ninja + +ndk:armeabi-v7a: + extends: .monado.ndk:build-base + +ndk:arm64-v8a: + extends: .monado.ndk:build-base + # Packaging +.monado.packaging.conditions: + rules: + # Only the default branch of the "upstream" repo. + - if: "$CI_PROJECT_PATH == $FDO_UPSTREAM_REPO && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH" + when: on_success + # Otherwise, don't build packages. + - when: never + .monado.base-job.debuild: + extends: + - .monado.packaging.conditions stage: package - only: - - *package_only_branch before_script: # Configure git - needed despite not actually committing here. - git config --global user.email "ryan.pavlik@collabora.com" @@ -273,7 +345,7 @@ debian:cmake:32bit: # Prep the source tree - git clean -dfx - git merge origin/${PACKAGE_BRANCH} --no-commit - - FULLNAME="Monado CI " debian/extra/prepare-commit-package.sh ${CI_COMMIT_SHA} 1~${BACKPORT_SUFFIX}~ci$(date --utc "+%Y%m%d") + - DEBFULLNAME="Monado CI" DEBEMAIL="ryan.pavlik@collabora.com" debian/extra/prepare-commit-package.sh ${CI_COMMIT_SHA} 1~${BACKPORT_SUFFIX}~ci$(date --utc "+%Y%m%d") # Build the package - debuild -uc -us # Use dput-ng to move the package-related files into some artifacts. @@ -308,16 +380,27 @@ ubuntu:focal:package: PACKAGE_BRANCH: ubuntu/focal DISTRO: focal +ubuntu:groovy:package: + extends: + - .monado.variables.ubuntu:groovy + - .fdo.distribution-image@ubuntu # from ci-templates + - .monado.base-job.debuild + + variables: + BACKPORT_SUFFIX: ubuntu20.04 + PACKAGE_BRANCH: ubuntu/focal + DISTRO: focal + reprepro:package: stage: reprepro - only: - - *package_only_branch extends: - .monado.variables.debian:buster + - .monado.packaging.conditions - .fdo.distribution-image@debian # from ci-templates dependencies: - debian:buster:package - ubuntu:focal:package + - ubuntu:groovy:package before_script: # Convince gnupg to work properly in CI - mkdir -p ~/.gnupg && chmod 700 ~/.gnupg @@ -353,33 +436,6 @@ reprepro:package: - "repo/" expire_in: 2 days -# Base of Android NDK builds. -# Takes the last :-delimited part of the name as the ABI to build for, -# so you don't need to do anything other than "extends" in the job -.monado.ndk:build-base: - stage: build - extends: - - .monado.variables.debian:buster-ndk - - .fdo.suffixed-image@debian # from ci-templates - variables: - ANDROID_PLATFORM: 26 - script: - - mkdir build - - pushd build - # This extracts the ABI from the job name - - export ABI=$(echo $CI_JOB_NAME | cut --delimiter=":" -f 2) - # Note we are pointing CMake to the host install of Eigen3 because it's header-only - # and thus this is safe to do. - - cmake -GNinja .. -DANDROID_PLATFORM=$ANDROID_PLATFORM -DANDROID_ABI=$ABI -DCMAKE_TOOLCHAIN_FILE=/opt/android-ndk/build/cmake/android.toolchain.cmake -DEigen3_DIR=/usr/lib/cmake/eigen3/ - - grep "^XRT_" CMakeCache.txt - - ninja - -ndk:armeabi-v7a: - extends: .monado.ndk:build-base - -ndk:arm64-v8a: - extends: .monado.ndk:build-base - ### # Pages ### diff --git a/.gitlab-ci/ubuntu_container_prep.sh b/.gitlab-ci/ubuntu_focal_container_prep.sh similarity index 100% rename from .gitlab-ci/ubuntu_container_prep.sh rename to .gitlab-ci/ubuntu_focal_container_prep.sh diff --git a/.gitlab-ci/ubuntu_groovy_container_prep.sh b/.gitlab-ci/ubuntu_groovy_container_prep.sh new file mode 100644 index 000000000..36480abd3 --- /dev/null +++ b/.gitlab-ci/ubuntu_groovy_container_prep.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Copyright 2021, Collabora, Ltd. and the Monado contributors +# SPDX-License-Identifier: BSL-1.0 + +# Nothing really needed. diff --git a/.reuse/dep5 b/.reuse/dep5 index 8616e6669..e2b525dc8 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -10,17 +10,11 @@ Files: doc/changes/drivers/* doc/changes/xrt/* doc/changes/auxiliary/* doc/changes/compositor/* -Copyright: 2020, Collabora, Ltd. and the Monado contributors + doc/changes/big/* +Copyright: 2020-2021, Collabora, Ltd. and the Monado contributors License: CC0-1.0 Comment: Prevents needing a license header per fragment between releases. -Files: src/external/flexkalman/.clang-format - src/external/flexkalman/flexkalman/README.md -Copyright: 2015, 2016, Sensics, Inc. - 2019, Collabora, Ltd. -License: Apache-2.0 -Comment: Copyright statement and license identifier missing. - Files: src/external/cjson/* Copyright: 2009-2017, Dave Gamble and cJSON contributors License: MIT @@ -32,14 +26,6 @@ Copyright: 2020, Two Blue Cubes Ltd. License: BSL-1.0 Comment: SPDX-License-Identifier missing. -Files: src/external/openxr_includes/loader_interfaces.h -Copyright: 2017, LunarG, Inc. - 2017, Valve Corporation - 2017-2019, The Khronos Group Inc. -License: Apache-2.0 -Comment: SPDX-License-Identifier missing. - - Files: src/external/jnipp/* Copyright: 2016-2020, Mitchell Dowd 2020, Collabora, Ltd. @@ -52,6 +38,11 @@ Copyright: 2016, mcximing License: BSD-2-Clause Comment: SPDX-License-Identifier missing. +Files: src/external/openvr_includes/* +Copyright: 2015-2020, Valve Corporation +License: BSD-3-Clause +Comment: License identifier missing. + Files: src/external/imgui/imgui* src/external/imgui/imconfig.h Copyright: 2014-2020, Omar Cornut diff --git a/CMakeLists.txt b/CMakeLists.txt index 8dcf94ab0..1a6ec17b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2018-2020, Collabora, Ltd. +# Copyright 2018-2021, Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 cmake_minimum_required(VERSION 3.10.2) @@ -12,10 +12,18 @@ endif() option(XRT_OPENXR_INSTALL_ABSOLUTE_RUNTIME_PATH "Use the absolute path to the runtime in the installed manifest, rather than a bare filename." ON) option(XRT_OPENXR_INSTALL_ACTIVE_RUNTIME "Make Monado the default OpenXR runtime on install" ON) +# We use C++17 +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# So that clangd/Intellisense/Sourcetrail know how to parse our code. +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + ### # Dependencies ### list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/sanitizers") include(CMakeDependentOption) include(SPIR-V) include(GNUInstallDirs) @@ -26,36 +34,46 @@ if(NOT ${CMAKE_VERSION} VERSION_LESS 3.9) check_ipo_supported(RESULT HAS_IPO) endif() -# Redundant mention of version is required because module defaults to looking for 2.91-compatible, -# which the config file for a 3.x says it's not compatible with. -find_package(Eigen3 3 REQUIRED) -find_package(Vulkan) -find_package(EGL) -find_package(HIDAPI) -find_package(OpenHMD) -find_package(OpenCV COMPONENTS core calib3d highgui imgproc imgcodecs features2d video CONFIG) -find_package(Libusb1) -find_package(JPEG) -find_package(realsense2 CONFIG) -find_package(SDL2 CONFIG) -find_package(ZLIB) -find_package(cJSON) -find_package(Systemd) -find_package(OpenGLES COMPONENTS V3) - # Android SDK doesn't look for 3.8 and 3.9, which is what new distros ship with. set(Python_ADDITIONAL_VERSIONS 3.8 3.9) if(NOT CMAKE_VERSION VERSION_LESS 3.12) find_package(Python3 REQUIRED Interpreter) set(PYTHON_EXECUTABLE Python3::Interpreter) else() - find_package(PythonInterp REQUIRED VERSION 3) + find_program(PYTHON_EXECUTABLE python3) if(PYTHON_EXECUTABLE MATCHES "WindowsApps") # If you hit this error, you will have to install Python 3 or try harder to tell CMake where it is. message(FATAL_ERROR "Found WindowsApps alias for Python. Make sure Python3 is installed, then choose 'Manage App Execution Aliases' in Start and disable the aliases for Python.") endif() endif() +# Redundant mention of version is required because module defaults to looking for 2.91-compatible, +# which the config file for a 3.x says it's not compatible with. +find_package(Eigen3 3 REQUIRED) +find_package(Vulkan MODULE) +find_package(EGL MODULE) +find_package(HIDAPI MODULE) +find_package(OpenHMD MODULE) +find_package(OpenCV COMPONENTS core calib3d highgui imgproc imgcodecs features2d video CONFIG) +find_package(Libusb1 MODULE) +find_package(JPEG MODULE) +find_package(realsense2 CONFIG) +find_package(depthai CONFIG) +find_package(SDL2 CONFIG) +find_package(ZLIB MODULE) +find_package(cJSON MODULE) +find_package(Systemd MODULE) +find_package(OpenGLES MODULE COMPONENTS V3) +find_package(LeapV2 MODULE) +find_package(ONNXRuntime MODULE) +find_package(Percetto MODULE) +if(NOT ANDROID) + find_package(PkgConfig MODULE) +endif() + +#https://github.com/arsenm/sanitizers-cmake +find_package(Sanitizers MODULE) + add_library(xrt-pthreads INTERFACE) if(WIN32) find_package(pthreads_windows REQUIRED) @@ -66,7 +84,7 @@ else() target_link_libraries(xrt-pthreads INTERFACE Threads::Threads) endif() -if(NOT ANDROID) +if(PKGCONFIG_FOUND AND NOT ANDROID) # @TODO Turn into a find_package LIBUVC file. pkg_check_modules(LIBUVC libuvc) @@ -79,7 +97,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(XRT_HAVE_LINUX YES) # Compositor backend find_package(X11) - find_package(PkgConfig) find_package(udev REQUIRED) set(XRT_HAVE_V4L2 TRUE) @@ -91,9 +108,11 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") pkg_search_module(WAYLAND wayland-client) pkg_search_module(WAYLAND_SCANNER wayland-scanner) pkg_search_module(WAYLAND_PROTOCOLS wayland-protocols) + pkg_search_module(LIBDRM libdrm) endif() find_package(OpenGL COMPONENTS GLX) pkg_search_module(DBUS dbus-1) + pkg_search_module(LIBBSD libbsd) pkg_check_modules(GST gstreamer-1.0 @@ -112,13 +131,31 @@ if(ANDROID) find_library(ANDROID_LOG_LIBRARY log) endif() +# Find a external SLAM implementation +set(EXTERNAL_SLAM_SYSTEMS kimera_vio) +foreach(slam_system IN LISTS EXTERNAL_SLAM_SYSTEMS) + if(PKGCONFIG_FOUND) + pkg_check_modules(${slam_system} ${slam_system}) + endif() + if(${slam_system}_FOUND) + set(SLAM ON) + set(SLAM_NAME ${slam_system}) + set(SLAM_LIBRARIES ${${slam_system}_LIBRARIES}) + set(SLAM_INCLUDE_DIRS ${${slam_system}_INCLUDE_DIRS}) + break() + endif() +endforeach() + # This one is named differently because that's what CTest uses option(BUILD_TESTING "Enable building of the test suite?" ON) option(XRT_FEATURE_COLOR_LOG "Enable logging in color on supported platforms" ON) +cmake_dependent_option(XRT_HAVE_PERCETTO "Enable percetto support" ON "PERCETTO_FOUND" OFF) +cmake_dependent_option(XRT_FEATURE_TRACING "Enable debug tracing on supported platforms" OFF "XRT_HAVE_PERCETTO" OFF) cmake_dependent_option(CMAKE_INTERPROCEDURAL_OPTIMIZATION "Enable inter-procedural (link-time) optimization" OFF "HAS_IPO" OFF) -cmake_dependent_option(XRT_HAVE_WAYLAND "Enable Wayland support" ON "WAYLAND_FOUND AND WAYLAND_SCANNER_FOUND AND WAYLAND_PROTOCOLS_FOUND" OFF) +cmake_dependent_option(XRT_HAVE_WAYLAND "Enable Wayland support" ON "WAYLAND_FOUND AND WAYLAND_SCANNER_FOUND AND WAYLAND_PROTOCOLS_FOUND AND LIBDRM_FOUND" OFF) +cmake_dependent_option(XRT_HAVE_WAYLAND_DIRECT "Enable Wayland direct support" ON "XRT_HAVE_WAYLAND AND LIBDRM_FOUND AND WAYLAND_PROTOCOLS_VERSION VERSION_GREATER_EQUAL 1.22" OFF) cmake_dependent_option(XRT_HAVE_XLIB "Enable xlib support" ON "X11_FOUND" OFF) cmake_dependent_option(XRT_HAVE_XRANDR "Enable xlib-xrandr support" ON "XRANDR_FOUND" OFF) cmake_dependent_option(XRT_HAVE_XCB "Enable xcb support" ON "XCB_FOUND" OFF) @@ -128,10 +165,10 @@ cmake_dependent_option(XRT_HAVE_OPENGL "Enable OpenGL Graphics API support" ON " cmake_dependent_option(XRT_HAVE_OPENGLES "Enable OpenGL-ES Graphics API support" ON "OpenGLES_FOUND" OFF) cmake_dependent_option(XRT_HAVE_EGL "Enable OpenGL on EGL Graphics API support" ON "EGL_FOUND; XRT_HAVE_OPENGL OR XRT_HAVE_OPENGLES" OFF) cmake_dependent_option(XRT_HAVE_DBUS "Enable dbus support (for BLE support)" ON "DBUS_FOUND" OFF) -cmake_dependent_option(XRT_HAVE_VF "Enable gstreamer support (for video file support)" ON "GST_FOUND" OFF) cmake_dependent_option(XRT_FEATURE_COMPOSITOR_MAIN "Build main compositor host functionality" ON "XRT_HAVE_VULKAN; XRT_HAVE_WAYLAND OR XRT_HAVE_XCB OR ANDROID OR WIN32" OFF) +cmake_dependent_option(XRT_HAVE_LIBBSD "Enable libbsd support" ON "LIBBSD_FOUND" OFF) cmake_dependent_option(XRT_FEATURE_OPENXR "Build OpenXR runtime target" ON "XRT_FEATURE_COMPOSITOR_MAIN" OFF) -cmake_dependent_option(XRT_FEATURE_SERVICE "Enable separate service module for OpenXR runtime" ON "NOT WIN32" OFF) +cmake_dependent_option(XRT_FEATURE_SERVICE "Enable separate service module for OpenXR runtime" ON "NOT WIN32 AND XRT_FEATURE_OPENXR" OFF) cmake_dependent_option(XRT_HAVE_SYSTEMD "Enable systemd support (for socket activation of service)" ON "Systemd_FOUND AND XRT_FEATURE_SERVICE" OFF) cmake_dependent_option(XRT_INSTALL_SYSTEMD_UNIT_FILES "Install user unit files for systemd socket activation on installation" ON "XRT_HAVE_SYSTEMD" OFF) cmake_dependent_option(XRT_INSTALL_ABSOLUTE_SYSTEMD_UNIT_FILES "Use an absolute path to monado-system in installed user unit files for systemd socket activation" ON "XRT_INSTALL_SYSTEMD_UNIT_FILES" OFF) @@ -172,30 +209,37 @@ cmake_dependent_option(XRT_HAVE_LIBUVC "Enable libuvc video driver" ON "LIBUVC_F cmake_dependent_option(XRT_HAVE_FFMPEG "Enable ffmpeg testing video driver" ON "FFMPEG_FOUND" OFF) cmake_dependent_option(XRT_HAVE_SDL2 "Enable use of SDL2" ON "SDL2_FOUND AND XRT_HAVE_OPENGL" OFF) cmake_dependent_option(XRT_HAVE_SYSTEM_CJSON "Enable cJSON from system, instead of bundled source" ON "CJSON_FOUND" OFF) - - +cmake_dependent_option(XRT_HAVE_GST "Enable gstreamer" ON "GST_FOUND" OFF) +cmake_dependent_option(XRT_HAVE_ONNXRUNTIME "Enable ONNX runtime support" ON "ONNXRUNTIME_FOUND" OFF) +cmake_dependent_option(XRT_HAVE_KIMERA_SLAM "Enable Kimera support" ON "kimera_vio_FOUND" OFF) +cmake_dependent_option(XRT_HAVE_SLAM "Enable SLAM tracking support" ON "SLAM;XRT_HAVE_OPENCV" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_PSVR "Enable PSVR HMD driver" ON "HIDAPI_FOUND" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_RS "Enable RealSense device driver" ON "realsense2_FOUND" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_VIVE "Enable driver for HTC Vive, Vive Pro, Valve Index, and their controllers" ON "ZLIB_FOUND AND XRT_HAVE_LINUX" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_OHMD "Enable OpenHMD driver" ON "OPENHMD_FOUND" OFF) -cmake_dependent_option(XRT_BUILD_DRIVER_HANDTRACKING "Enable Camera Hand Tracking driver" ON "XRT_HAVE_V4L2" OFF) +cmake_dependent_option(XRT_BUILD_DRIVER_HANDTRACKING "Enable Camera Hand Tracking driver" ON "XRT_HAVE_ONNXRUNTIME AND XRT_HAVE_OPENCV AND XRT_HAVE_V4L2" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_DAYDREAM "Enable the Google Daydream View controller driver (BLE, via D-Bus)" ON "XRT_HAVE_DBUS" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_ARDUINO "Enable Arduino input device with BLE via via D-Bus" ON "XRT_HAVE_DBUS" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_ILLIXR "Enable ILLIXR driver" ON "ILLIXR_PATH" OFF) - option(XRT_BUILD_DRIVER_DUMMY "Enable dummy driver" ON) +cmake_dependent_option(XRT_BUILD_DRIVER_ULV2 "Enable Ultraleap v2 driver" ON "LeapV2_FOUND" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_REMOTE "Enable remote debugging driver" ON "XRT_HAVE_LINUX OR ANDROID" OFF) +option(XRT_BUILD_DRIVER_WMR "Enable Windows Mixed Reality driver" ON) # These all use the Monado internal hid wrapper. cmake_dependent_option(XRT_BUILD_DRIVER_HDK "Enable HDK driver" ON "XRT_HAVE_INTERNAL_HID" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_PSMV "Enable Playstation Move driver" ON "XRT_HAVE_INTERNAL_HID" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_HYDRA "Enable Hydra driver" ON "XRT_HAVE_INTERNAL_HID" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_NS "Enable North Star driver" ON "XRT_HAVE_INTERNAL_HID" OFF) +cmake_dependent_option(XRT_BUILD_DRIVER_VF "Build video frame driver (for video file support, uses gstreamer)" ON "XRT_HAVE_GST" OFF) +cmake_dependent_option(XRT_BUILD_DRIVER_DEPTHAI "DepthAI" ON "depthai_FOUND" OFF) # This one defaults to off, even if we find the deps. cmake_dependent_option(XRT_BUILD_DRIVER_SURVIVE "Enable libsurvive driver" ON "SURVIVE_FOUND" OFF) cmake_dependent_option(XRT_BUILD_DRIVER_ANDROID "Enable Android sensors driver" ON "ANDROID" OFF) +cmake_dependent_option(XRT_BUILD_DRIVER_QWERTY "Enable Qwerty driver" ON "XRT_HAVE_SDL2" OFF) +cmake_dependent_option(XRT_BUILD_DRIVER_EUROC "Enable EuRoC dataset driver for SLAM evaluation" ON "XRT_HAVE_OPENCV" OFF) # You can set this from a superproject to add a driver # All drivers must be listed in here to be included in the generated header! @@ -216,7 +260,13 @@ list(APPEND AVAILABLE_DRIVERS "REMOTE" "SURVIVE" "V4L2" + "ULV2" + "VF" + "DEPTHAI" "VIVE" + "QWERTY" + "WMR" + "EUROC" ) @@ -290,6 +340,17 @@ if(CMAKE_INTERPROCEDURAL_OPTIMIZATION) message(STATUS "Inter-procedural optimization enabled") endif() +# Make sure we have pretty colours +option (FORCE_COLORED_OUTPUT "Always produce ANSI-colored output (GNU/Clang only)." FALSE) + +if ("${FORCE_COLORED_OUTPUT}") + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + add_compile_options (-fdiagnostics-color=always) + elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + add_compile_options (-fcolor-diagnostics) + endif () +endif () + ### # Decend into madness. ### @@ -298,30 +359,37 @@ add_subdirectory(src) add_subdirectory(doc) if(BUILD_TESTING) - include(CTest) - add_subdirectory(tests) + include(CTest) + add_subdirectory(tests) endif() message(STATUS "#####----- Config -----#####") -message(STATUS "# GIT_DESC: ${GIT_DESC}") +message(STATUS "# GIT_DESC: ${GIT_DESC}") message(STATUS "#") -message(STATUS "# WAYLAND: ${XRT_HAVE_WAYLAND}") -message(STATUS "# XLIB: ${XRT_HAVE_XLIB}") -message(STATUS "# XRANDR: ${XRT_HAVE_XRANDR}") -message(STATUS "# XCB: ${XRT_HAVE_XCB}") -message(STATUS "# OPENGL: ${XRT_HAVE_OPENGL}") -message(STATUS "# OPENGLES: ${XRT_HAVE_OPENGLES}") -message(STATUS "# VULKAN: ${XRT_HAVE_VULKAN}") -message(STATUS "# EGL: ${XRT_HAVE_EGL}") -message(STATUS "# DBUS: ${XRT_HAVE_DBUS}") -message(STATUS "# VF: ${XRT_HAVE_VF}") -message(STATUS "# LIBUSB: ${XRT_HAVE_LIBUSB}") -message(STATUS "# JPEG: ${XRT_HAVE_JPEG}") -message(STATUS "# OPENCV: ${XRT_HAVE_OPENCV}") -message(STATUS "# LIBUVC: ${XRT_HAVE_LIBUVC}") -message(STATUS "# FFMPEG: ${XRT_HAVE_FFMPEG}") -message(STATUS "# SDL2: ${XRT_HAVE_SDL2}") -message(STATUS "# SYSTEM_CJSON: ${XRT_HAVE_SYSTEM_CJSON}") +message(STATUS "# GST (GStreamer): ${XRT_HAVE_GST}") +message(STATUS "# WAYLAND: ${XRT_HAVE_WAYLAND}") +message(STATUS "# WAYLAND_DIRECT: ${XRT_HAVE_WAYLAND_DIRECT}") +message(STATUS "# XLIB: ${XRT_HAVE_XLIB}") +message(STATUS "# XRANDR: ${XRT_HAVE_XRANDR}") +message(STATUS "# XCB: ${XRT_HAVE_XCB}") +message(STATUS "# VULKAN: ${XRT_HAVE_VULKAN}") +message(STATUS "# OPENGL: ${XRT_HAVE_OPENGL}") +message(STATUS "# OPENGLES: ${XRT_HAVE_OPENGLES}") +message(STATUS "# EGL: ${XRT_HAVE_EGL}") +message(STATUS "# DBUS: ${XRT_HAVE_DBUS}") +message(STATUS "# LIBBSD: ${XRT_HAVE_LIBBSD}") +message(STATUS "# SYSTEMD: ${XRT_HAVE_SYSTEMD}") +message(STATUS "# LIBUSB: ${XRT_HAVE_LIBUSB}") +message(STATUS "# JPEG: ${XRT_HAVE_JPEG}") +message(STATUS "# OPENCV: ${XRT_HAVE_OPENCV}") +message(STATUS "# LIBUVC: ${XRT_HAVE_LIBUVC}") +message(STATUS "# FFMPEG: ${XRT_HAVE_FFMPEG}") +message(STATUS "# SDL2: ${XRT_HAVE_SDL2}") +message(STATUS "# PERCETTO: ${XRT_HAVE_PERCETTO}") +message(STATUS "# ONNXRUNTIME: ${XRT_HAVE_ONNXRUNTIME}") +message(STATUS "# SYSTEM_CJSON: ${XRT_HAVE_SYSTEM_CJSON}") +message(STATUS "# KIMERA: ${XRT_HAVE_KIMERA_SLAM}") +message(STATUS "# SLAM: ${XRT_HAVE_SLAM}") message(STATUS "#") message(STATUS "# FEATURE_COMPOSITOR_MAIN: ${XRT_FEATURE_COMPOSITOR_MAIN}") message(STATUS "# FEATURE_SERVICE: ${XRT_FEATURE_SERVICE}") @@ -332,6 +400,8 @@ message(STATUS "# FEATURE_OPENXR_LAYER_CYLINDER: ${XRT_FEATURE_OPENXR_ message(STATUS "# FEATURE_OPENXR_LAYER_EQUIRECT2: ${XRT_FEATURE_OPENXR_LAYER_EQUIRECT2}") message(STATUS "# FEATURE_OPENXR_LAYER_EQUIRECT1: ${XRT_FEATURE_OPENXR_LAYER_EQUIRECT1}") message(STATUS "# FEATURE_STEAMVR_PLUGIN: ${XRT_FEATURE_STEAMVR_PLUGIN}") +message(STATUS "# FEATURE_COLOR_LOG: ${XRT_FEATURE_COLOR_LOG}") +message(STATUS "# FEATURE_TRACING: ${XRT_FEATURE_TRACING}") message(STATUS "#") message(STATUS "# DRIVER_ANDROID: ${XRT_BUILD_DRIVER_ANDROID}") message(STATUS "# DRIVER_ARDUINO: ${XRT_BUILD_DRIVER_ARDUINO}") @@ -342,11 +412,21 @@ message(STATUS "# DRIVER_HDK: ${XRT_BUILD_DRIVER_HDK}") message(STATUS "# DRIVER_HYDRA: ${XRT_BUILD_DRIVER_HYDRA}") message(STATUS "# DRIVER_ILLIXR: ${XRT_BUILD_DRIVER_ILLIXR}") message(STATUS "# DRIVER_NS: ${XRT_BUILD_DRIVER_NS}") +message(STATUS "# DRIVER_ULV2: ${XRT_BUILD_DRIVER_ULV2}") message(STATUS "# DRIVER_OHMD: ${XRT_BUILD_DRIVER_OHMD}") message(STATUS "# DRIVER_PSMV: ${XRT_BUILD_DRIVER_PSMV}") message(STATUS "# DRIVER_PSVR: ${XRT_BUILD_DRIVER_PSVR}") message(STATUS "# DRIVER_RS: ${XRT_BUILD_DRIVER_RS}") message(STATUS "# DRIVER_REMOTE: ${XRT_BUILD_DRIVER_REMOTE}") message(STATUS "# DRIVER_SURVIVE: ${XRT_BUILD_DRIVER_SURVIVE}") +message(STATUS "# DRIVER_VF: ${XRT_BUILD_DRIVER_VF}") +message(STATUS "# DRIVER_DEPTHAI: ${XRT_BUILD_DRIVER_DEPTHAI}") message(STATUS "# DRIVER_VIVE: ${XRT_BUILD_DRIVER_VIVE}") +message(STATUS "# DRIVER_QWERTY: ${XRT_BUILD_DRIVER_QWERTY}") +message(STATUS "# DRIVER_WMR: ${XRT_BUILD_DRIVER_WMR}") +message(STATUS "# DRIVER_EUROC: ${XRT_BUILD_DRIVER_EUROC}") message(STATUS "#####----- Config -----#####") + +if(XRT_FEATURE_SERVICE AND NOT XRT_FEATURE_OPENXR) + message(FATAL_ERROR "XRT_FEATURE_SERVICE requires XRT_FEATURE_OPENXR to be enabled") +endif() diff --git a/LICENSES/Unlicense.txt b/LICENSES/Unlicense.txt new file mode 100644 index 000000000..cde4ac698 --- /dev/null +++ b/LICENSES/Unlicense.txt @@ -0,0 +1,10 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. + +In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/README.md b/README.md index 4b9cc6765..829ebd381 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,10 @@ best with the following vcpkg packages installed: * pthreads eigen3 libusb hidapi zlib doxygen +If you have a recent [vcpkg](https://vcpkg.io) installed and use the appropriate +CMake toolchain file, the vcpkg manifest in the Monado repository will instruct +vcpkg to locally install the dependencies automatically. + Tested distributions that are fully compatible, on Intel (Vulkan only) and AMD graphics (Vulkan and OpenGL): @@ -241,8 +245,11 @@ scripts/format-project.sh You can optionally put something like `CLANG_FORMAT=clang-format-7` before that command if your clang-format binary isn't named `clang-format`. -Note that you'll typically prefer to use something like `git clang-format` +**Note that you'll typically prefer** to use something like `git clang-format` to just re-format your changes, in case version differences in tools result in overall format changes. +The CI "style" job currently runs on Debian Buster, so it has clang-format-7. +We will probably update that job to Bullseye or Ubuntu 20.10, which will allow +using clang-format-11 by default, soon. [OpenHMD]: http://openhmd.net [drm-lease]: https://haagch.frickel.club/#!drmlease%2Emd diff --git a/build.gradle b/build.gradle index 91a25088f..99f043386 100644 --- a/build.gradle +++ b/build.gradle @@ -1,29 +1,30 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 buildscript { ext { - kotlinVersion = '1.4.10' - latestAboutLibsRelease = "8.5.0" + kotlinVersion = '1.5.0' + + // Skip 8.8.6 - see https://github.com/mikepenz/AboutLibraries/issues/648 + latestAboutLibsRelease = '8.8.5' androidxCoreVersion = "1.3.2" - androidxAnnotationVersion = "1.1.0" + androidxAnnotationVersion = '1.2.0' androidxAppCompatVersion = "1.2.0" androidxLifecycleVersion = "2.2.0" androidxConstraintLayoutVersion = '2.0.4' - hiltVersion = "2.29.1-alpha" + hiltVersion = '2.35.1' - // Saw some breakage when updating to 1.2? - materialVersion = "1.1.0" + materialVersion = "1.3.0" } repositories { google() - jcenter() mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:4.1.1' + classpath 'com.android.tools.build:gradle:4.2.1' + //noinspection DifferentKotlinGradleVersion classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion" classpath "com.mikepenz.aboutlibraries.plugin:aboutlibraries-plugin:${latestAboutLibsRelease}" classpath 'com.quittle:svg-2-android-vector:0.0.5' @@ -59,6 +60,6 @@ ext { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/cmake/FindEGL.cmake b/cmake/FindEGL.cmake index 160009277..1ac539dc1 100644 --- a/cmake/FindEGL.cmake +++ b/cmake/FindEGL.cmake @@ -36,7 +36,7 @@ #============================================================================= # Copyright 2014 Alex Merry # Copyright 2014 Martin Gräßlin -# Copyright 2019 Ryan Pavlik +# Copyright 2019, 2021 Ryan Pavlik # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -67,8 +67,12 @@ include(CMakePushCheckState) # Use pkg-config to get the directories and then use these values # in the FIND_PATH() and FIND_LIBRARY() calls -find_package(PkgConfig) -pkg_check_modules(PKG_EGL QUIET egl) +if(NOT ANDROID) + find_package(PkgConfig QUIET) + if(PKGCONFIG_FOUND) + pkg_check_modules(PKG_EGL QUIET egl) + endif() +endif() set(EGL_DEFINITIONS ${PKG_EGL_CFLAGS_OTHER}) diff --git a/cmake/FindHIDAPI.cmake b/cmake/FindHIDAPI.cmake index 202194497..3d97b8113 100644 --- a/cmake/FindHIDAPI.cmake +++ b/cmake/FindHIDAPI.cmake @@ -40,11 +40,11 @@ # ``HIDAPI_LIBRARIES`` # # Original Author: -# 2009-2010, 2019 Ryan Pavlik +# 2009-2010, 2019, 2021 Ryan Pavlik # http://academic.cleardefinition.com # # Copyright Iowa State University 2009-2010. -# Copyright Collabora, Ltd. 2019. +# Copyright Collabora, Ltd. 2019, 2021. # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. # (See accompanying file LICENSE_1_0.txt or copy at @@ -72,17 +72,19 @@ if(NOT HIDAPI_FIND_COMPONENTS) endif() # Ask pkg-config for hints -find_package(PkgConfig QUIET) -if(PKG_CONFIG_FOUND) - set(_old_prefix_path "${CMAKE_PREFIX_PATH}") - # So pkg-config uses HIDAPI_ROOT_DIR too. - if(HIDAPI_ROOT_DIR) - list(APPEND CMAKE_PREFIX_PATH ${HIDAPI_ROOT_DIR}) +if(NOT ANDROID) + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + set(_old_prefix_path "${CMAKE_PREFIX_PATH}") + # So pkg-config uses HIDAPI_ROOT_DIR too. + if(HIDAPI_ROOT_DIR) + list(APPEND CMAKE_PREFIX_PATH ${HIDAPI_ROOT_DIR}) + endif() + pkg_check_modules(PC_HIDAPI_LIBUSB QUIET hidapi-libusb) + pkg_check_modules(PC_HIDAPI_HIDRAW QUIET hidapi-hidraw) + # Restore + set(CMAKE_PREFIX_PATH "${_old_prefix_path}") endif() - pkg_check_modules(PC_HIDAPI_LIBUSB QUIET hidapi-libusb) - pkg_check_modules(PC_HIDAPI_HIDRAW QUIET hidapi-hidraw) - # Restore - set(CMAKE_PREFIX_PATH "${_old_prefix_path}") endif() # Actually search diff --git a/cmake/FindLeapV2.cmake b/cmake/FindLeapV2.cmake new file mode 100644 index 000000000..d914a62d3 --- /dev/null +++ b/cmake/FindLeapV2.cmake @@ -0,0 +1,62 @@ +# Copyright 2019-2021, Collabora, Ltd. +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) +# +# Original Author: +# 2021 Moses Turner + +#.rst: +# FindLeapV2 +# --------------- +# +# Find the Ultraleap v2 drivers +# +# Targets +# ^^^^^^^ +# +# If successful, the following import target is created. +# +# ``LeapV2::LeapV2`` +# +# Cache variables +# ^^^^^^^^^^^^^^^ +# +# The following cache variable may also be set to assist/control the operation of this module: +# +# ``LeapV2_ROOT_DIR`` +# The root to search for Leap v2. +# + +set(LeapV2_ROOT_DIR + "${LeapV2_ROOT_DIR}" + CACHE PATH "Root to search for LeapV2") + +find_path( + LeapV2_INCLUDE_DIR + NAMES Leap.h LeapMath.h + PATHS ${LeapV2_ROOT_DIR} + PATH_SUFFIXES include) +find_library( + LeapV2_LIBRARY + NAMES Leap + PATHS ${LeapV2_ROOT_DIR} + PATH_SUFFIXES lib lib/x64) +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LeapV2 REQUIRED_VARS LeapV2_INCLUDE_DIR + LeapV2_LIBRARY) +if(LeapV2_FOUND) + set(LeapV2_INCLUDE_DIRS "${LeapV2_INCLUDE_DIR}") + set(LeapV2_LIBRARIES "${LeapV2_LIBRARY}") + if(NOT TARGET LeapV2::LeapV2) + add_library(LeapV2::LeapV2 UNKNOWN IMPORTED) + endif() + set_target_properties( + LeapV2::LeapV2 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${LeapV2_INCLUDE_DIR}") + set_target_properties(LeapV2::LeapV2 PROPERTIES IMPORTED_LOCATION + "${LeapV2_LIBRARY}") + mark_as_advanced(LeapV2_INCLUDE_DIR LeapV2_LIBRARY) +endif() +mark_as_advanced(LeapV2_ROOT_DIR) diff --git a/cmake/FindLibcheck.cmake b/cmake/FindLibcheck.cmake index af1d3af56..25b583c21 100644 --- a/cmake/FindLibcheck.cmake +++ b/cmake/FindLibcheck.cmake @@ -1,11 +1,11 @@ -# Copyright 2019 Collabora, Ltd. +# Copyright 2019, 2021 Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. # (See accompanying file LICENSE_1_0.txt or copy at # http://www.boost.org/LICENSE_1_0.txt) # # Original Author: -# 2019 Ryan Pavlik +# 2019, 2021 Ryan Pavlik #.rst: # FindCheck @@ -36,17 +36,20 @@ set(LIBCHECK_ROOT_DIR "${LIBCHECK_ROOT_DIR}" CACHE PATH "Root to search for libcheck") -find_package(PkgConfig QUIET) -if(PKG_CONFIG_FOUND) - set(_old_prefix_path "${CMAKE_PREFIX_PATH}") - # So pkg-config uses LIBCHECK_ROOT_DIR too. - if(LIBCHECK_ROOT_DIR) - list(APPEND CMAKE_PREFIX_PATH ${LIBCHECK_ROOT_DIR}) +if(NOT ANDROID) + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + set(_old_prefix_path "${CMAKE_PREFIX_PATH}") + # So pkg-config uses LIBCHECK_ROOT_DIR too. + if(LIBCHECK_ROOT_DIR) + list(APPEND CMAKE_PREFIX_PATH ${LIBCHECK_ROOT_DIR}) + endif() + pkg_check_modules(PC_LIBCHECK QUIET check) + # Restore + set(CMAKE_PREFIX_PATH "${_old_prefix_path}") endif() - pkg_check_modules(PC_LIBCHECK QUIET check) - # Restore - set(CMAKE_PREFIX_PATH "${_old_prefix_path}") endif() + find_path( LIBCHECK_INCLUDE_DIR NAMES check.h diff --git a/cmake/FindLibusb1.cmake b/cmake/FindLibusb1.cmake index f1c63e324..ea5aae1b4 100644 --- a/cmake/FindLibusb1.cmake +++ b/cmake/FindLibusb1.cmake @@ -14,11 +14,11 @@ # FindPackageHandleStandardArgs (known included with CMake >=2.6.2) # # Original Author: -# 2009-2010 Ryan Pavlik -# http://academic.cleardefinition.com -# Iowa State University HCI Graduate Program/VRAC +# 2009-2010, 2021 Ryan Pavlik # # Copyright Iowa State University 2009-2010. +# Copyright Collabora, Ltd 2021. +# # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. # (See accompanying file LICENSE_1_0.txt or copy at @@ -47,9 +47,11 @@ if(WIN32) endif() else() set(_lib_suffixes) - find_package(PkgConfig QUIET) - if(PKG_CONFIG_FOUND) - pkg_check_modules(PC_LIBUSB1 libusb-1.0) + if(NOT ANDROID) + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + pkg_check_modules(PC_LIBUSB1 QUIET libusb-1.0) + endif() endif() endif() diff --git a/cmake/FindONNXRuntime.cmake b/cmake/FindONNXRuntime.cmake new file mode 100644 index 000000000..d9c17e44e --- /dev/null +++ b/cmake/FindONNXRuntime.cmake @@ -0,0 +1,78 @@ +# Copyright 2021, Collabora, Ltd. +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) +# +# Original Author: +# 2021 Moses Turner +# 2021 Ryan Pavlik + +#.rst: +# FindONNXRuntime +# --------------- +# +# Find the ONNX runtime +# +# Targets +# ^^^^^^^ +# +# If successful, the following import target is created. +# +# ``ONNXRuntime::ONNXRuntime`` +# +# Cache variables +# ^^^^^^^^^^^^^^^ +# +# The following cache variable may also be set to assist/control the operation of this module: +# +# ``ONNXRuntime_ROOT_DIR`` +# The root to search for ONNX runtime. +# + +include(FeatureSummary) +set_package_properties( + ONNXRuntime PROPERTIES + URL "https://onnxruntime.ai/" + DESCRIPTION "Machine learning runtime") + +set(ONNXRuntime_ROOT_DIR + "${ONNXRuntime_ROOT_DIR}" + CACHE PATH "Root to search for ONNXRuntime") + +find_package(PkgConfig) +pkg_check_modules(PC_ONNXRuntime QUIET libonnxruntime) + +find_library( + ONNXRuntime_LIBRARY + NAMES onnxruntime + PATHS ${ONNXRuntime_ROOT_DIR} + PATH_SUFFIXES lib + HINTS ${PC_ONNXRuntime_LIBRARY_DIRS}) +find_path( + ONNXRuntime_INCLUDE_DIR core/session/onnxruntime_cxx_api.h + PATHS ${ONNXRuntime_ROOT_DIR} + PATH_SUFFIXES onnxruntime include/onnxruntime + HINTS ${PC_ONNXRuntime_INCLUDE_DIRS}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + ONNXRuntime REQUIRED_VARS ONNXRuntime_INCLUDE_DIR ONNXRuntime_LIBRARY) + +if(ONNXRuntime_FOUND) + set(ONNXRuntime_INCLUDE_DIRS ${ONNXRuntime_INCLUDE_DIR}) + set(ONNXRuntime_LIBRARIES "${ONNXRuntime_LIBRARY}") + if(NOT TARGET ONNXRuntime::ONNXRuntime) + add_library(ONNXRuntime::ONNXRuntime UNKNOWN IMPORTED) + endif() + set_target_properties( + ONNXRuntime::ONNXRuntime PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${ONNXRuntime_INCLUDE_DIRS}") + set_target_properties( + ONNXRuntime::ONNXRuntime + PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${ONNXRuntime_LIBRARY}") + mark_as_advanced(ONNXRuntime_INCLUDE_DIRS ONNXRuntime_LIBRARY) +endif() + +mark_as_advanced(ONNXRuntime_ROOT_DIR) diff --git a/cmake/FindOpenGLES.cmake b/cmake/FindOpenGLES.cmake index fc8aeb56c..b16a1d33a 100644 --- a/cmake/FindOpenGLES.cmake +++ b/cmake/FindOpenGLES.cmake @@ -1,11 +1,11 @@ -# Copyright 2020 Collabora, Ltd. +# Copyright 2020-2021 Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. # (See accompanying file LICENSE_1_0.txt or copy at # http://www.boost.org/LICENSE_1_0.txt) # # Original Author: -# 2020 Ryan Pavlik +# 2020-2021 Ryan Pavlik #[[.rst: FindOpenGLES @@ -53,18 +53,22 @@ set(OpenGLES_ROOT_DIR if(NOT OpenGLES_FIND_COMPONENTS) set(OpenGLES_FIND_COMPONENTS V2) endif() -find_package(PkgConfig QUIET) -if(PKG_CONFIG_FOUND) - set(_old_prefix_path "${CMAKE_PREFIX_PATH}") - # So pkg-config uses OpenGLES_ROOT_DIR too. - if(OpenGLES_ROOT_DIR) - list(APPEND CMAKE_PREFIX_PATH ${OpenGLES_ROOT_DIR}) + +if(NOT ANDROID) + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + set(_old_prefix_path "${CMAKE_PREFIX_PATH}") + # So pkg-config uses OpenGLES_ROOT_DIR too. + if(OpenGLES_ROOT_DIR) + list(APPEND CMAKE_PREFIX_PATH ${OpenGLES_ROOT_DIR}) + endif() + pkg_check_modules(PC_glesv1_cm QUIET glesv1_cm) + pkg_check_modules(PC_glesv2 QUIET glesv2) + # Restore + set(CMAKE_PREFIX_PATH "${_old_prefix_path}") endif() - pkg_check_modules(PC_glesv1_cm QUIET glesv1_cm) - pkg_check_modules(PC_glesv2 QUIET glesv2) - # Restore - set(CMAKE_PREFIX_PATH "${_old_prefix_path}") endif() + find_path( OpenGLES_V1_INCLUDE_DIR NAMES GLES/gl.h diff --git a/cmake/FindOpenHMD.cmake b/cmake/FindOpenHMD.cmake index 556cc7879..ea36f2e91 100644 --- a/cmake/FindOpenHMD.cmake +++ b/cmake/FindOpenHMD.cmake @@ -1,11 +1,11 @@ -# Copyright 2019 Collabora, Ltd. +# Copyright 2019, 2021 Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. # (See accompanying file LICENSE_1_0.txt or copy at # http://www.boost.org/LICENSE_1_0.txt) # # Original Author: -# 2019 Ryan Pavlik +# 2019, 2021 Ryan Pavlik #.rst: # FindOpenHMD @@ -34,16 +34,18 @@ set(OPENHMD_ROOT_DIR "${OPENHMD_ROOT_DIR}" CACHE PATH "Root to search for OpenHMD") -find_package(PkgConfig QUIET) -if(PKG_CONFIG_FOUND) - set(_old_prefix_path "${CMAKE_PREFIX_PATH}") - # So pkg-config uses OPENHMD_ROOT_DIR too. - if(OPENHMD_ROOT_DIR) - list(APPEND CMAKE_PREFIX_PATH ${OPENHMD_ROOT_DIR}) +if(NOT ANDROID) + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + set(_old_prefix_path "${CMAKE_PREFIX_PATH}") + # So pkg-config uses OPENHMD_ROOT_DIR too. + if(OPENHMD_ROOT_DIR) + list(APPEND CMAKE_PREFIX_PATH ${OPENHMD_ROOT_DIR}) + endif() + pkg_check_modules(PC_OPENHMD QUIET openhmd) + # Restore + set(CMAKE_PREFIX_PATH "${_old_prefix_path}") endif() - pkg_check_modules(PC_OPENHMD QUIET openhmd) - # Restore - set(CMAKE_PREFIX_PATH "${_old_prefix_path}") endif() find_path( diff --git a/cmake/FindPercetto.cmake b/cmake/FindPercetto.cmake new file mode 100644 index 000000000..616a977c3 --- /dev/null +++ b/cmake/FindPercetto.cmake @@ -0,0 +1,114 @@ +# Copyright 2021 Collabora, Ltd. +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) +# +# Original Author: +# 2021 Ryan Pavlik + +#[[.rst: +FindPercetto +--------------- + +Find the Percetto C wrapper around the Perfetto tracing API. + +Targets +^^^^^^^ + +If successful, the following imported targets are created. + +* ``Percetto::percetto`` + +Cache variables +^^^^^^^^^^^^^^^ + +The following cache variable may also be set to assist/control the operation of this module: + +``Percetto_ROOT_DIR`` + The root to search for Percetto. +#]] + +set(Percetto_ROOT_DIR + "${Percetto_ROOT_DIR}" + CACHE PATH "Root to search for Percetto") + +include(FeatureSummary) +set_package_properties( + Percetto PROPERTIES + URL "https://github.com/olvaffe/percetto/" + DESCRIPTION "A C wrapper around the C++ Perfetto tracing SDK.") + +# See if we can find something made by android prefab (gradle) +find_package(Percetto QUIET CONFIG NAMES percetto Percetto) +if(Percetto_FOUND) + if(TARGET Percetto::percetto) + # OK, good - unexpected, but good. + get_target_property(Percetto_LIBRARY Percetto::percetto + IMPORTED_LOCATION) + get_target_property(Percetto_INCLUDE_DIR Percetto::percetto + INTERFACE_INCLUDE_DIRECTORIES) + elseif(TARGET percetto::percetto) + # Let's make our own of the right name + add_library(Percetto::percetto STATIC IMPORTED) + get_target_property(Percetto_INCLUDE_DIR percetto::percetto + INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(Percetto_LIBRARY percetto::percetto + IMPORTED_LOCATION) + set_target_properties( + Percetto::percetto + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${Percetto_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION ${Percetto_LIBRARY}) + else() + message(FATAL_ERROR "assumptions failed") + endif() + find_package_handle_standard_args( + Percetto REQUIRED_VARS Percetto_LIBRARY Percetto_INCLUDE_DIR) + return() +endif() + +if(NOT ANDROID) + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + set(_old_prefix_path "${CMAKE_PREFIX_PATH}") + # So pkg-config uses Percetto_ROOT_DIR too. + if(Percetto_ROOT_DIR) + list(APPEND CMAKE_PREFIX_PATH ${Percetto_ROOT_DIR}) + endif() + pkg_check_modules(PC_percetto QUIET percetto) + # Restore + set(CMAKE_PREFIX_PATH "${_old_prefix_path}") + endif() +endif() + +find_path( + Percetto_INCLUDE_DIR + NAMES percetto.h + PATHS ${Percetto_ROOT_DIR} + HINTS ${PC_percetto_INCLUDE_DIRS} + PATH_SUFFIXES include) + +find_library( + Percetto_LIBRARY + NAMES percetto + PATHS ${Percetto_ROOT_DIR} + HINTS ${PC_percetto_LIBRARY_DIRS} + PATH_SUFFIXES lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Percetto REQUIRED_VARS Percetto_INCLUDE_DIR + Percetto_LIBRARY) +if(Percetto_FOUND) + if(NOT TARGET Percetto::percetto) + add_library(Percetto::percetto STATIC IMPORTED) + + set_target_properties( + Percetto::percetto + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${Percetto_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION ${Percetto_LIBRARY}) + endif() + mark_as_advanced(Percetto_LIBRARY Percetto_INCLUDE_DIR) +endif() +mark_as_advanced(Percetto_ROOT_DIR) diff --git a/cmake/FindSystemd.cmake b/cmake/FindSystemd.cmake index 2580d848e..b41b37169 100644 --- a/cmake/FindSystemd.cmake +++ b/cmake/FindSystemd.cmake @@ -15,7 +15,7 @@ # #============================================================================= # Copyright (c) 2015 Jari Vetoniemi -# Copyright (c) 2020 Collabora, Ltd. +# Copyright (c) 2020-2021 Collabora, Ltd. # # Distributed under the OSI-approved BSD License (the "License"); # @@ -31,8 +31,13 @@ set_package_properties( URL "http://freedesktop.org/wiki/Software/systemd/" DESCRIPTION "System and Service Manager") -find_package(PkgConfig) -pkg_check_modules(PC_SYSTEMD QUIET libsystemd) +if(NOT ANDROID) + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + pkg_check_modules(PC_SYSTEMD QUIET libsystemd) + endif() +endif() + find_library( SYSTEMD_LIBRARY NAMES systemd diff --git a/cmake/Findudev.cmake b/cmake/Findudev.cmake index ef8cea39c..6ad89ea54 100644 --- a/cmake/Findudev.cmake +++ b/cmake/Findudev.cmake @@ -13,8 +13,13 @@ # Requires these CMake modules: # FindPackageHandleStandardArgs (known included with CMake >=2.6.2) # -# Original Author: -# Copyright 2014 Kevin M. Godby +# Original Authors: +# 2014 Kevin M. Godby +# 2021 Ryan Pavlik +# +# Copyright 2014, Kevin M. Godby +# Copyright 2021, Collabora, Ltd. +# # SPDX-License-Identifier: BSL-1.0 # # Distributed under the Boost Software License, Version 1.0. @@ -27,9 +32,11 @@ set(UDEV_ROOT_DIR PATH "Directory to search for udev") -find_package(PkgConfig QUIET) -if(PKG_CONFIG_FOUND) - pkg_check_modules(PC_LIBUDEV libudev) +if(NOT ANDROID) + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + pkg_check_modules(PC_LIBUDEV QUIET libudev) + endif() endif() find_library(UDEV_LIBRARY diff --git a/cmake/GetGitRevisionDescription.cmake b/cmake/GetGitRevisionDescription.cmake index db291a9b2..4fbd90db7 100644 --- a/cmake/GetGitRevisionDescription.cmake +++ b/cmake/GetGitRevisionDescription.cmake @@ -3,7 +3,7 @@ # These functions force a re-configure on each git commit so that you can # trust the values of the variables in your build system. # -# get_git_head_revision( [ ...]) +# get_git_head_revision( [ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR]) # # Returns the refspec and sha hash of the current head revision # @@ -68,7 +68,7 @@ function(_git_find_closest_git_dir _start_dir _git_dir_var) while(NOT EXISTS "${git_dir}") # .git dir not found, search parent directories set(git_previous_parent "${cur_dir}") - get_filename_component(cur_dir ${cur_dir} DIRECTORY) + get_filename_component(cur_dir "${cur_dir}" DIRECTORY) if(cur_dir STREQUAL git_previous_parent) # We have reached the root directory, we are not in git set(${_git_dir_var} @@ -86,10 +86,15 @@ endfunction() function(get_git_head_revision _refspecvar _hashvar) _git_find_closest_git_dir("${CMAKE_CURRENT_SOURCE_DIR}" GIT_DIR) + if("${ARGN}" STREQUAL "ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR") + set(ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR TRUE) + else() + set(ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR FALSE) + endif() if(NOT "${GIT_DIR}" STREQUAL "") file(RELATIVE_PATH _relative_to_source_dir "${CMAKE_SOURCE_DIR}" "${GIT_DIR}") - if("${_relative_to_source_dir}" MATCHES "[.][.]") + if("${_relative_to_source_dir}" MATCHES "[.][.]" AND NOT ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR) # We've gone above the CMake root dir. set(GIT_DIR "") endif() diff --git a/cmake/sanitizers/FindASan.cmake b/cmake/sanitizers/FindASan.cmake new file mode 100644 index 000000000..9a6d0221a --- /dev/null +++ b/cmake/sanitizers/FindASan.cmake @@ -0,0 +1,61 @@ +# The MIT License (MIT) +# +# Copyright (c) +# 2013 Matthew Arsenault +# 2015-2016 RWTH Aachen University, Federal Republic of Germany +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +option(SANITIZE_ADDRESS "Enable AddressSanitizer for sanitized targets." Off) + +set(FLAG_CANDIDATES + # Clang 3.2+ use this version. The no-omit-frame-pointer option is optional. + "-g -fsanitize=address -fno-omit-frame-pointer" + "-g -fsanitize=address" + + # Older deprecated flag for ASan + "-g -faddress-sanitizer" +) + + +if (SANITIZE_ADDRESS AND (SANITIZE_THREAD OR SANITIZE_MEMORY)) + message(FATAL_ERROR "AddressSanitizer is not compatible with " + "ThreadSanitizer or MemorySanitizer.") +endif () + + +include(sanitize-helpers) + +if (SANITIZE_ADDRESS) + sanitizer_check_compiler_flags("${FLAG_CANDIDATES}" "AddressSanitizer" + "ASan") + + find_program(ASan_WRAPPER "asan-wrapper" PATHS ${CMAKE_MODULE_PATH}) + mark_as_advanced(ASan_WRAPPER) +endif () + +function (add_sanitize_address TARGET) + if (NOT SANITIZE_ADDRESS) + return() + endif () + + sanitizer_add_flags(${TARGET} "AddressSanitizer" "ASan") +endfunction () diff --git a/cmake/sanitizers/FindMSan.cmake b/cmake/sanitizers/FindMSan.cmake new file mode 100644 index 000000000..8972535d2 --- /dev/null +++ b/cmake/sanitizers/FindMSan.cmake @@ -0,0 +1,59 @@ +# The MIT License (MIT) +# +# Copyright (c) +# 2013 Matthew Arsenault +# 2015-2016 RWTH Aachen University, Federal Republic of Germany +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +option(SANITIZE_MEMORY "Enable MemorySanitizer for sanitized targets." Off) + +set(FLAG_CANDIDATES + "-g -fsanitize=memory" +) + + +include(sanitize-helpers) + +if (SANITIZE_MEMORY) + if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + message(WARNING "MemorySanitizer disabled for target ${TARGET} because " + "MemorySanitizer is supported for Linux systems only.") + set(SANITIZE_MEMORY Off CACHE BOOL + "Enable MemorySanitizer for sanitized targets." FORCE) + elseif (NOT ${CMAKE_SIZEOF_VOID_P} EQUAL 8) + message(WARNING "MemorySanitizer disabled for target ${TARGET} because " + "MemorySanitizer is supported for 64bit systems only.") + set(SANITIZE_MEMORY Off CACHE BOOL + "Enable MemorySanitizer for sanitized targets." FORCE) + else () + sanitizer_check_compiler_flags("${FLAG_CANDIDATES}" "MemorySanitizer" + "MSan") + endif () +endif () + +function (add_sanitize_memory TARGET) + if (NOT SANITIZE_MEMORY) + return() + endif () + + sanitizer_add_flags(${TARGET} "MemorySanitizer" "MSan") +endfunction () diff --git a/cmake/sanitizers/FindSanitizers.cmake b/cmake/sanitizers/FindSanitizers.cmake new file mode 100755 index 000000000..a393f163b --- /dev/null +++ b/cmake/sanitizers/FindSanitizers.cmake @@ -0,0 +1,96 @@ +# The MIT License (MIT) +# +# Copyright (c) +# 2013 Matthew Arsenault +# 2015-2016 RWTH Aachen University, Federal Republic of Germany +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +# If any of the used compiler is a GNU compiler, add a second option to static +# link against the sanitizers. +option(SANITIZE_LINK_STATIC "Try to link static against sanitizers." Off) + + + + +set(FIND_QUIETLY_FLAG "") +if (DEFINED Sanitizers_FIND_QUIETLY) + set(FIND_QUIETLY_FLAG "QUIET") +endif () + +find_package(ASan ${FIND_QUIETLY_FLAG}) +find_package(TSan ${FIND_QUIETLY_FLAG}) +find_package(MSan ${FIND_QUIETLY_FLAG}) +find_package(UBSan ${FIND_QUIETLY_FLAG}) + + + + +function(sanitizer_add_blacklist_file FILE) + if(NOT IS_ABSOLUTE ${FILE}) + set(FILE "${CMAKE_CURRENT_SOURCE_DIR}/${FILE}") + endif() + get_filename_component(FILE "${FILE}" REALPATH) + + sanitizer_check_compiler_flags("-fsanitize-blacklist=${FILE}" + "SanitizerBlacklist" "SanBlist") +endfunction() + +function(add_sanitizers ...) + # If no sanitizer is enabled, return immediately. + if (NOT (SANITIZE_ADDRESS OR SANITIZE_MEMORY OR SANITIZE_THREAD OR + SANITIZE_UNDEFINED)) + return() + endif () + + foreach (TARGET ${ARGV}) + # Check if this target will be compiled by exactly one compiler. Other- + # wise sanitizers can't be used and a warning should be printed once. + get_target_property(TARGET_TYPE ${TARGET} TYPE) + if (TARGET_TYPE STREQUAL "INTERFACE_LIBRARY") + message(WARNING "Can't use any sanitizers for target ${TARGET}, " + "because it is an interface library and cannot be " + "compiled directly.") + return() + endif () + sanitizer_target_compilers(${TARGET} TARGET_COMPILER) + list(LENGTH TARGET_COMPILER NUM_COMPILERS) + if (NUM_COMPILERS GREATER 1) + message(WARNING "Can't use any sanitizers for target ${TARGET}, " + "because it will be compiled by incompatible compilers. " + "Target will be compiled without sanitizers.") + return() + + # If the target is compiled by no or no known compiler, give a warning. + elseif (NUM_COMPILERS EQUAL 0) + message(WARNING "Sanitizers for target ${TARGET} may not be" + " usable, because it uses no or an unknown compiler. " + "This is a false warning for targets using only " + "object lib(s) as input.") + endif () + + # Add sanitizers for target. + add_sanitize_address(${TARGET}) + add_sanitize_thread(${TARGET}) + add_sanitize_memory(${TARGET}) + add_sanitize_undefined(${TARGET}) + endforeach () +endfunction(add_sanitizers) diff --git a/cmake/sanitizers/FindTSan.cmake b/cmake/sanitizers/FindTSan.cmake new file mode 100644 index 000000000..4ccd6ea49 --- /dev/null +++ b/cmake/sanitizers/FindTSan.cmake @@ -0,0 +1,67 @@ +# The MIT License (MIT) +# +# Copyright (c) +# 2013 Matthew Arsenault +# 2015-2016 RWTH Aachen University, Federal Republic of Germany +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +option(SANITIZE_THREAD "Enable ThreadSanitizer for sanitized targets." Off) + +set(FLAG_CANDIDATES + "-g -fsanitize=thread" +) + + +# ThreadSanitizer is not compatible with MemorySanitizer. +if (SANITIZE_THREAD AND SANITIZE_MEMORY) + message(FATAL_ERROR "ThreadSanitizer is not compatible with " + "MemorySanitizer.") +endif () + + +include(sanitize-helpers) + +if (SANITIZE_THREAD) + if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Linux" AND + NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") + message(WARNING "ThreadSanitizer disabled for target ${TARGET} because " + "ThreadSanitizer is supported for Linux systems and macOS only.") + set(SANITIZE_THREAD Off CACHE BOOL + "Enable ThreadSanitizer for sanitized targets." FORCE) + elseif (NOT ${CMAKE_SIZEOF_VOID_P} EQUAL 8) + message(WARNING "ThreadSanitizer disabled for target ${TARGET} because " + "ThreadSanitizer is supported for 64bit systems only.") + set(SANITIZE_THREAD Off CACHE BOOL + "Enable ThreadSanitizer for sanitized targets." FORCE) + else () + sanitizer_check_compiler_flags("${FLAG_CANDIDATES}" "ThreadSanitizer" + "TSan") + endif () +endif () + +function (add_sanitize_thread TARGET) + if (NOT SANITIZE_THREAD) + return() + endif () + + sanitizer_add_flags(${TARGET} "ThreadSanitizer" "TSan") +endfunction () diff --git a/cmake/sanitizers/FindUBSan.cmake b/cmake/sanitizers/FindUBSan.cmake new file mode 100644 index 000000000..5eabf683a --- /dev/null +++ b/cmake/sanitizers/FindUBSan.cmake @@ -0,0 +1,48 @@ +# The MIT License (MIT) +# +# Copyright (c) +# 2013 Matthew Arsenault +# 2015-2016 RWTH Aachen University, Federal Republic of Germany +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +option(SANITIZE_UNDEFINED + "Enable UndefinedBehaviorSanitizer for sanitized targets." Off) + +set(FLAG_CANDIDATES + "-g -fsanitize=undefined" +) + + +include(sanitize-helpers) + +if (SANITIZE_UNDEFINED) + sanitizer_check_compiler_flags("${FLAG_CANDIDATES}" + "UndefinedBehaviorSanitizer" "UBSan") +endif () + +function (add_sanitize_undefined TARGET) + if (NOT SANITIZE_UNDEFINED) + return() + endif () + + sanitizer_add_flags(${TARGET} "UndefinedBehaviorSanitizer" "UBSan") +endfunction () diff --git a/cmake/sanitizers/asan-wrapper b/cmake/sanitizers/asan-wrapper new file mode 100755 index 000000000..2f133fa28 --- /dev/null +++ b/cmake/sanitizers/asan-wrapper @@ -0,0 +1,57 @@ +#!/bin/sh + +# The MIT License (MIT) +# +# Copyright (c) +# 2013 Matthew Arsenault +# 2015-2016 RWTH Aachen University, Federal Republic of Germany +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +# This script is a wrapper for AddressSanitizer. In some special cases you need +# to preload AddressSanitizer to avoid error messages - e.g. if you're +# preloading another library to your application. At the moment this script will +# only do something, if we're running on a Linux platform. OSX might not be +# affected. + + +# Exit immediately, if platform is not Linux. +if [ "$(uname)" != "Linux" ] +then + exec $@ +fi + + +# Get the used libasan of the application ($1). If a libasan was found, it will +# be prepended to LD_PRELOAD. +libasan=$(ldd $1 | grep libasan | sed "s/^[[:space:]]//" | cut -d' ' -f1) +if [ -n "$libasan" ] +then + if [ -n "$LD_PRELOAD" ] + then + export LD_PRELOAD="$libasan:$LD_PRELOAD" + else + export LD_PRELOAD="$libasan" + fi +fi + +# Execute the application. +exec $@ diff --git a/cmake/sanitizers/sanitize-helpers.cmake b/cmake/sanitizers/sanitize-helpers.cmake new file mode 100755 index 000000000..f0d8d9e98 --- /dev/null +++ b/cmake/sanitizers/sanitize-helpers.cmake @@ -0,0 +1,179 @@ +# The MIT License (MIT) +# +# Copyright (c) +# 2013 Matthew Arsenault +# 2015-2016 RWTH Aachen University, Federal Republic of Germany +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +# Helper function to get the language of a source file. +function (sanitizer_lang_of_source FILE RETURN_VAR) + get_filename_component(LONGEST_EXT "${FILE}" EXT) + # If extension is empty return. This can happen for extensionless headers + if("${LONGEST_EXT}" STREQUAL "") + set(${RETURN_VAR} "" PARENT_SCOPE) + return() + endif() + # Get shortest extension as some files can have dot in their names + string(REGEX REPLACE "^.*(\\.[^.]+)$" "\\1" FILE_EXT ${LONGEST_EXT}) + string(TOLOWER "${FILE_EXT}" FILE_EXT) + string(SUBSTRING "${FILE_EXT}" 1 -1 FILE_EXT) + + get_property(ENABLED_LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) + foreach (LANG ${ENABLED_LANGUAGES}) + list(FIND CMAKE_${LANG}_SOURCE_FILE_EXTENSIONS "${FILE_EXT}" TEMP) + if (NOT ${TEMP} EQUAL -1) + set(${RETURN_VAR} "${LANG}" PARENT_SCOPE) + return() + endif () + endforeach() + + set(${RETURN_VAR} "" PARENT_SCOPE) +endfunction () + + +# Helper function to get compilers used by a target. +function (sanitizer_target_compilers TARGET RETURN_VAR) + # Check if all sources for target use the same compiler. If a target uses + # e.g. C and Fortran mixed and uses different compilers (e.g. clang and + # gfortran) this can trigger huge problems, because different compilers may + # use different implementations for sanitizers. + set(BUFFER "") + get_target_property(TSOURCES ${TARGET} SOURCES) + foreach (FILE ${TSOURCES}) + # If expression was found, FILE is a generator-expression for an object + # library. Object libraries will be ignored. + string(REGEX MATCH "TARGET_OBJECTS:([^ >]+)" _file ${FILE}) + if ("${_file}" STREQUAL "") + sanitizer_lang_of_source(${FILE} LANG) + if (LANG) + list(APPEND BUFFER ${CMAKE_${LANG}_COMPILER_ID}) + endif () + endif () + endforeach () + + list(REMOVE_DUPLICATES BUFFER) + set(${RETURN_VAR} "${BUFFER}" PARENT_SCOPE) +endfunction () + + +# Helper function to check compiler flags for language compiler. +function (sanitizer_check_compiler_flag FLAG LANG VARIABLE) + if (${LANG} STREQUAL "C") + include(CheckCCompilerFlag) + check_c_compiler_flag("${FLAG}" ${VARIABLE}) + + elseif (${LANG} STREQUAL "CXX") + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("${FLAG}" ${VARIABLE}) + + elseif (${LANG} STREQUAL "Fortran") + # CheckFortranCompilerFlag was introduced in CMake 3.x. To be compatible + # with older Cmake versions, we will check if this module is present + # before we use it. Otherwise we will define Fortran coverage support as + # not available. + include(CheckFortranCompilerFlag OPTIONAL RESULT_VARIABLE INCLUDED) + if (INCLUDED) + check_fortran_compiler_flag("${FLAG}" ${VARIABLE}) + elseif (NOT CMAKE_REQUIRED_QUIET) + message(STATUS "Performing Test ${VARIABLE}") + message(STATUS "Performing Test ${VARIABLE}" + " - Failed (Check not supported)") + endif () + endif() +endfunction () + + +# Helper function to test compiler flags. +function (sanitizer_check_compiler_flags FLAG_CANDIDATES NAME PREFIX) + set(CMAKE_REQUIRED_QUIET ${${PREFIX}_FIND_QUIETLY}) + + get_property(ENABLED_LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) + foreach (LANG ${ENABLED_LANGUAGES}) + # Sanitizer flags are not dependend on language, but the used compiler. + # So instead of searching flags foreach language, search flags foreach + # compiler used. + set(COMPILER ${CMAKE_${LANG}_COMPILER_ID}) + if (NOT DEFINED ${PREFIX}_${COMPILER}_FLAGS) + foreach (FLAG ${FLAG_CANDIDATES}) + if(NOT CMAKE_REQUIRED_QUIET) + message(STATUS "Try ${COMPILER} ${NAME} flag = [${FLAG}]") + endif() + + set(CMAKE_REQUIRED_FLAGS "${FLAG}") + unset(${PREFIX}_FLAG_DETECTED CACHE) + sanitizer_check_compiler_flag("${FLAG}" ${LANG} + ${PREFIX}_FLAG_DETECTED) + + if (${PREFIX}_FLAG_DETECTED) + # If compiler is a GNU compiler, search for static flag, if + # SANITIZE_LINK_STATIC is enabled. + if (SANITIZE_LINK_STATIC AND (${COMPILER} STREQUAL "GNU")) + string(TOLOWER ${PREFIX} PREFIX_lower) + sanitizer_check_compiler_flag( + "-static-lib${PREFIX_lower}" ${LANG} + ${PREFIX}_STATIC_FLAG_DETECTED) + + if (${PREFIX}_STATIC_FLAG_DETECTED) + set(FLAG "-static-lib${PREFIX_lower} ${FLAG}") + endif () + endif () + + set(${PREFIX}_${COMPILER}_FLAGS "${FLAG}" CACHE STRING + "${NAME} flags for ${COMPILER} compiler.") + mark_as_advanced(${PREFIX}_${COMPILER}_FLAGS) + break() + endif () + endforeach () + + if (NOT ${PREFIX}_FLAG_DETECTED) + set(${PREFIX}_${COMPILER}_FLAGS "" CACHE STRING + "${NAME} flags for ${COMPILER} compiler.") + mark_as_advanced(${PREFIX}_${COMPILER}_FLAGS) + + message(WARNING "${NAME} is not available for ${COMPILER} " + "compiler. Targets using this compiler will be " + "compiled without ${NAME}.") + endif () + endif () + endforeach () +endfunction () + + +# Helper to assign sanitizer flags for TARGET. +function (sanitizer_add_flags TARGET NAME PREFIX) + # Get list of compilers used by target and check, if sanitizer is available + # for this target. Other compiler checks like check for conflicting + # compilers will be done in add_sanitizers function. + sanitizer_target_compilers(${TARGET} TARGET_COMPILER) + list(LENGTH TARGET_COMPILER NUM_COMPILERS) + if ("${${PREFIX}_${TARGET_COMPILER}_FLAGS}" STREQUAL "") + return() + endif() + + # Set compile- and link-flags for target. + set_property(TARGET ${TARGET} APPEND_STRING + PROPERTY COMPILE_FLAGS " ${${PREFIX}_${TARGET_COMPILER}_FLAGS}") + set_property(TARGET ${TARGET} APPEND_STRING + PROPERTY COMPILE_FLAGS " ${SanBlist_${TARGET_COMPILER}_FLAGS}") + set_property(TARGET ${TARGET} APPEND_STRING + PROPERTY LINK_FLAGS " ${${PREFIX}_${TARGET_COMPILER}_FLAGS}") +endfunction () diff --git a/debian/README.source b/debian/README.source index b1be94f46..6020e1ed4 100644 --- a/debian/README.source +++ b/debian/README.source @@ -4,7 +4,7 @@ directly within the "upstream" source tree. There is useful stuff, more directed at package maintenance for distro submission, in the Debian copy of this file: - + To build a package for local use: diff --git a/debian/changelog b/debian/changelog index 53b603a89..18f135b79 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,11 +4,20 @@ monado (21.0.0~dfsg1-2~bpo10+1) buster-backports; urgency=medium -- Ryan Pavlik Tue, 23 Feb 2021 13:40:07 -0600 -monado (21.0.0~dfsg1-2) UNRELEASED; urgency=medium +monado (21.0.0~dfsg1-2) unstable; urgency=medium - * d/control: Add Build-Depends on libbsd-dev for pidfile support when building service. + * d/control + - Add Build-Depends on libbsd-dev for pidfile support when + building service. + - Bump Standards-Version to 4.6.0, no changes required. + - Remove Build-Depends that we don't actually use. + - Exclude some dependencies on less-common arches to fix builds. + * d/copyright: Update + * Backport patch for upstream to fix FTBFS, closes: #997239 + * Clean up/annotate patches + * Switch to pandoc from markdown for formatting changelog. - -- Ryan Pavlik Thu, 15 Apr 2021 16:51:09 -0500 + -- Ryan Pavlik Tue, 26 Oct 2021 16:59:16 -0500 monado (21.0.0~dfsg1-1) unstable; urgency=medium diff --git a/debian/control b/debian/control index 4eaa70609..96ad67b00 100644 --- a/debian/control +++ b/debian/control @@ -5,10 +5,8 @@ Section: libs Priority: optional Build-Depends: debhelper-compat (= 12), cmake, - doxygen, glslang-tools, - graphviz, - libavcodec-dev, + libavcodec-dev [!hppa !sh4], libbsd-dev [linux-any], libcjson-dev, libdbus-1-dev [linux-any], @@ -19,7 +17,7 @@ Build-Depends: debhelper-compat (= 12), libgstreamer1.0-dev, libgstreamer-plugins-base1.0-dev, libhidapi-dev [!hurd-i386], - libopencv-dev, + libopencv-dev [!alpha !ia64 !sparc64 !x32], libsystemd-dev [linux-any], libsdl2-dev, libudev-dev, @@ -33,9 +31,9 @@ Build-Depends: debhelper-compat (= 12), libxcb-randr0-dev, libxrandr-dev, libxxf86vm-dev, - markdown , + pandoc , pkg-config -Standards-Version: 4.5.0 +Standards-Version: 4.6.0 Vcs-Browser: https://gitlab.freedesktop.org/monado/monado/-/tree/debian/buster-backports Vcs-Git: https://gitlab.freedesktop.org/monado/monado.git -b debian/buster-backports Homepage: https://monado.freedesktop.org/ diff --git a/debian/copyright b/debian/copyright index c68593c34..32582d215 100644 --- a/debian/copyright +++ b/debian/copyright @@ -5,7 +5,7 @@ Copyright: 2018-2021, Collabora, Ltd. License: BSL-1.0 Files: * -Copyright: 2018-2020, Collabora, Ltd. +Copyright: 2018-2021, Collabora, Ltd. License: BSL-1.0 Files: cmake/GetGitRevisionDescription.cmake.in diff --git a/debian/copyright-scan-patterns.yml b/debian/copyright-scan-patterns.yml index 1f2375f2b..4df1898bb 100644 --- a/debian/copyright-scan-patterns.yml +++ b/debian/copyright-scan-patterns.yml @@ -3,4 +3,3 @@ ignore: pattern: - .reuse/* - .gitlab-ci - diff --git a/debian/rules b/debian/rules index 8fe65ce08..a384fdee0 100755 --- a/debian/rules +++ b/debian/rules @@ -46,7 +46,7 @@ override_dh_install: ifeq (,$(findstring nodoc,$(DEB_BUILD_OPTIONS))) mkdir -p debian/tmp/usr/share/doc/monado cp doc/CHANGELOG.md debian/tmp/usr/share/doc/monado/changelog - markdown doc/CHANGELOG.md > debian/tmp/usr/share/doc/monado/changelog.html + pandoc -f markdown -t html doc/CHANGELOG.md > debian/tmp/usr/share/doc/monado/changelog.html endif ifneq (,$(findstring linux,$(DEB_HOST_ARCH_OS))) sed -i "s/%N/monado/" debian/tmp/usr/lib/systemd/user/monado.service diff --git a/doc/CHANGELOG.md b/doc/CHANGELOG.md index d456dfaca..1047bfbb8 100644 --- a/doc/CHANGELOG.md +++ b/doc/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changelog for Monado +# Changelog for Monado {#CHANGELOG} ```txt SPDX-License-Identifier: CC0-1.0 diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index dff5e83ff..d6b248ba7 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -29,6 +29,14 @@ if(BUILD_DOC) # request to configure the file configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY) + # copy the schema + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/example_configs/config_v0.schema.json + ${CMAKE_CURRENT_BINARY_DIR}/html/config_v0.schema.json + @ONLY) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/example_configs/config_v0.schema.json.license + ${CMAKE_CURRENT_BINARY_DIR}/html/config_v0.schema.json.license + @ONLY) + # note the option ALL which allows to build the docs together with the application add_custom_target(doc_doxygen ALL COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT} diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index f1801aef1..710d5f147 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -1,4 +1,4 @@ -# Copyright 2018-2020, Collabora, Ltd. and the Monado contributors +# Copyright 2018-2021, Collabora, Ltd. and the Monado contributors # SPDX-License-Identifier: BSL-1.0 QUIET = YES @@ -17,6 +17,8 @@ RECURSIVE = YES EXCLUDE = @SRCDIR@/src/external \ @SRCDIR@/doc/changes \ @BUILDDIR@ +EXCLUDE_PATTERNS = */build/* + STRIP_FROM_PATH = @SRCDIR@/src/xrt \ @SRCDIR@/src/xrt/include \ @SRCDIR@/doc @@ -63,6 +65,7 @@ ALWAYS_DETAILED_SEC = YES WARN_IF_UNDOCUMENTED = @DOXYGEN_WARN_UNDOCUMENTED@ EXTRACT_ALL = @DOXYGEN_EXTRACT_ALL@ HIDE_UNDOC_RELATIONS = NO +EXTRACT_STATIC = @DOXYGEN_EXTRACT_ALL@ MACRO_EXPANSION = YES diff --git a/doc/changes/.markdownlint.yaml b/doc/changes/.markdownlint.yaml new file mode 100644 index 000000000..fc50c2457 --- /dev/null +++ b/doc/changes/.markdownlint.yaml @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: CC0-1.0 +# SPDX-FileCopyrightText: 2021 Collabora, Ltd. + +# Changelog fragments will never start with a header. +MD041: false diff --git a/doc/changes/auxiliary/mr.717.md b/doc/changes/auxiliary/mr.717.md new file mode 100644 index 000000000..48faabaa2 --- /dev/null +++ b/doc/changes/auxiliary/mr.717.md @@ -0,0 +1 @@ +m/3dof: Add assert to catch time traveling drivers. diff --git a/doc/changes/auxiliary/mr.721.1.md b/doc/changes/auxiliary/mr.721.1.md new file mode 100644 index 000000000..0d29ad43f --- /dev/null +++ b/doc/changes/auxiliary/mr.721.1.md @@ -0,0 +1 @@ +u/time: Add helper comparison functions. diff --git a/doc/changes/auxiliary/mr.721.2.md b/doc/changes/auxiliary/mr.721.2.md new file mode 100644 index 000000000..8721d09f0 --- /dev/null +++ b/doc/changes/auxiliary/mr.721.2.md @@ -0,0 +1 @@ +vulkan: Add fence import function. diff --git a/doc/changes/auxiliary/mr.721.3.md b/doc/changes/auxiliary/mr.721.3.md new file mode 100644 index 000000000..d24c9ee60 --- /dev/null +++ b/doc/changes/auxiliary/mr.721.3.md @@ -0,0 +1,3 @@ +u/timing: A rather large refactor that turns makes the rendering timing helper +be more like the frame timing helper. This also makes the rendering timing +adjust the frame timing of the app so that latency is reduced. diff --git a/doc/changes/auxiliary/mr.735.md b/doc/changes/auxiliary/mr.735.md new file mode 100644 index 000000000..f0086fbe1 --- /dev/null +++ b/doc/changes/auxiliary/mr.735.md @@ -0,0 +1 @@ +math: Fixes for M_PI on Windows. diff --git a/doc/changes/auxiliary/mr.810.md b/doc/changes/auxiliary/mr.810.md new file mode 100644 index 000000000..ce1c15eea --- /dev/null +++ b/doc/changes/auxiliary/mr.810.md @@ -0,0 +1 @@ +Move C++-only functionality into the newly-conventional namespaces. diff --git a/doc/changes/auxiliary/mr.811.md b/doc/changes/auxiliary/mr.811.md new file mode 100644 index 000000000..94c8ca8fa --- /dev/null +++ b/doc/changes/auxiliary/mr.811.md @@ -0,0 +1,4 @@ +--- +- mr.840 +--- +u/trace_marker: Switch from homegrown tracing code to using Percetto/Perfetto. diff --git a/doc/changes/auxiliary/mr.825.md b/doc/changes/auxiliary/mr.825.md new file mode 100644 index 000000000..f6e4f62a5 --- /dev/null +++ b/doc/changes/auxiliary/mr.825.md @@ -0,0 +1,2 @@ +t/fm: Add simple FrameMat that wraps a cv::Mat, this allows us to easily pass +cv::Mat's around without the C code needing to know about OpenCV. diff --git a/doc/changes/auxiliary/mr.839.1.md b/doc/changes/auxiliary/mr.839.1.md new file mode 100644 index 000000000..d3a39d013 --- /dev/null +++ b/doc/changes/auxiliary/mr.839.1.md @@ -0,0 +1,4 @@ +util: added `u_device_2d_extents` and `u_setup_2d_extents_split_side_by_side` - +this is hopefully to eliminate confusion: the FOV you had to give to +`u_device_split_side_by_side` was just a dummy value, but some people thought +it was the actual headset's FOV. diff --git a/doc/changes/auxiliary/mr.839.2.md b/doc/changes/auxiliary/mr.839.2.md new file mode 100644 index 000000000..e506dc6e4 --- /dev/null +++ b/doc/changes/auxiliary/mr.839.2.md @@ -0,0 +1 @@ +math: Add `math_map_ranges` function, does the same thing as Arduino's `map`. diff --git a/doc/changes/auxiliary/mr.841.1.md b/doc/changes/auxiliary/mr.841.1.md new file mode 100644 index 000000000..6652ce241 --- /dev/null +++ b/doc/changes/auxiliary/mr.841.1.md @@ -0,0 +1 @@ +vulkan: Add more functions to vk_bundle struct. diff --git a/doc/changes/auxiliary/mr.841.2.md b/doc/changes/auxiliary/mr.841.2.md new file mode 100644 index 000000000..605595004 --- /dev/null +++ b/doc/changes/auxiliary/mr.841.2.md @@ -0,0 +1 @@ +vulkan: Make it possible to create a compute only queue. diff --git a/doc/changes/auxiliary/mr.841.3.md b/doc/changes/auxiliary/mr.841.3.md new file mode 100644 index 000000000..421140d9e --- /dev/null +++ b/doc/changes/auxiliary/mr.841.3.md @@ -0,0 +1 @@ +vulkan: Refactor and tidy extension handling. diff --git a/doc/changes/auxiliary/mr.841.4.md b/doc/changes/auxiliary/mr.841.4.md new file mode 100644 index 000000000..79ac635fb --- /dev/null +++ b/doc/changes/auxiliary/mr.841.4.md @@ -0,0 +1 @@ +vulkan: Add support for VK_EXT_robustness2 diff --git a/doc/changes/auxiliary/mr.841.5.md b/doc/changes/auxiliary/mr.841.5.md new file mode 100644 index 000000000..686118636 --- /dev/null +++ b/doc/changes/auxiliary/mr.841.5.md @@ -0,0 +1 @@ +vulkan: Add code to handle optional device features. diff --git a/doc/changes/auxiliary/mr.859.1.md b/doc/changes/auxiliary/mr.859.1.md new file mode 100644 index 000000000..e07fcc458 --- /dev/null +++ b/doc/changes/auxiliary/mr.859.1.md @@ -0,0 +1,2 @@ +t/calibration: Add support for RGB image streams, also add a special sink +converter helper to handle this case. diff --git a/doc/changes/auxiliary/mr.859.2.md b/doc/changes/auxiliary/mr.859.2.md new file mode 100644 index 000000000..dab9ac7c5 --- /dev/null +++ b/doc/changes/auxiliary/mr.859.2.md @@ -0,0 +1 @@ +t/calibration: Make it possible to select number distortion parameters. diff --git a/doc/changes/auxiliary/mr.911.md b/doc/changes/auxiliary/mr.911.md new file mode 100644 index 000000000..93be1a4e0 --- /dev/null +++ b/doc/changes/auxiliary/mr.911.md @@ -0,0 +1 @@ +t/calibration: Add support for findChessboardCornersSB in calibration code. diff --git a/doc/changes/big/mr.774.md b/doc/changes/big/mr.774.md new file mode 100644 index 000000000..3a2e186b9 --- /dev/null +++ b/doc/changes/big/mr.774.md @@ -0,0 +1,2 @@ +New WinMR driver, the initial commit only adds simple 3DoF support and not +distortion support. diff --git a/doc/changes/big/mr.873.md b/doc/changes/big/mr.873.md new file mode 100644 index 000000000..b809da40f --- /dev/null +++ b/doc/changes/big/mr.873.md @@ -0,0 +1,2 @@ +New compute based rendering backend in the compositor, currently only supports +a single projection layer so it's not enabled by default. But comes with ATW. diff --git a/doc/changes/compositor/mr.677.md b/doc/changes/compositor/mr.677.md new file mode 100644 index 000000000..d73ea57ba --- /dev/null +++ b/doc/changes/compositor/mr.677.md @@ -0,0 +1 @@ +comp: Fix layer submission on nvidia tegra. diff --git a/doc/changes/compositor/mr.697.md b/doc/changes/compositor/mr.697.md new file mode 100644 index 000000000..00529e4a7 --- /dev/null +++ b/doc/changes/compositor/mr.697.md @@ -0,0 +1 @@ +main: Integrate new frame timing code. diff --git a/doc/changes/compositor/mr.705.md b/doc/changes/compositor/mr.705.md new file mode 100644 index 000000000..5393b659b --- /dev/null +++ b/doc/changes/compositor/mr.705.md @@ -0,0 +1,2 @@ +client: Handle EGL_NO_CONTEXT_KHR gracefully if the EGLDisplay supports +EGL_KHR_no_config_context. diff --git a/doc/changes/compositor/mr.721.md b/doc/changes/compositor/mr.721.md new file mode 100644 index 000000000..d5e50e7dc --- /dev/null +++ b/doc/changes/compositor/mr.721.md @@ -0,0 +1,8 @@ +--- +- mr.754 +- mr.759 +--- +multi: Introduce a new multi client compositor layer, this allows rendering code +to be moved from the IPC layer into the compositor, separating concerns. The +main compositor always uses the multi client compositor, as it gives us a async +render loop for free. diff --git a/doc/changes/compositor/mr.767.md b/doc/changes/compositor/mr.767.md new file mode 100644 index 000000000..f3e74cab2 --- /dev/null +++ b/doc/changes/compositor/mr.767.md @@ -0,0 +1,6 @@ +--- +- issue.120 +- mr.787 +--- +main: Make it possible to create the swapchain later when actually needed, +and have the swapchain be in a non-ready state that stops drawing. diff --git a/doc/changes/compositor/mr.827.md b/doc/changes/compositor/mr.827.md new file mode 100644 index 000000000..d5aac18e0 --- /dev/null +++ b/doc/changes/compositor/mr.827.md @@ -0,0 +1,2 @@ +client: Use the EGL compositor's display in swapchain, previously it tried to +use the current one, which when running on a new thread would explode. diff --git a/doc/changes/compositor/mr.833.md b/doc/changes/compositor/mr.833.md new file mode 100644 index 000000000..21a60b52c --- /dev/null +++ b/doc/changes/compositor/mr.833.md @@ -0,0 +1,3 @@ +main: Do not list VK_FORMAT_A2B10G10R10_UNORM_PACK32 as a supported format, it's +not enough to show linear colours without banding but isn't used that often so +do not list it. diff --git a/doc/changes/compositor/mr.841.1.md b/doc/changes/compositor/mr.841.1.md new file mode 100644 index 000000000..ab3fbac82 --- /dev/null +++ b/doc/changes/compositor/mr.841.1.md @@ -0,0 +1 @@ +render: Prepare for submitting work on the compute queue. diff --git a/doc/changes/compositor/mr.841.2.md b/doc/changes/compositor/mr.841.2.md new file mode 100644 index 000000000..4e9159623 --- /dev/null +++ b/doc/changes/compositor/mr.841.2.md @@ -0,0 +1 @@ +main: Prepare for submitting work on the compute queue. diff --git a/doc/changes/compositor/mr.841.3.md b/doc/changes/compositor/mr.841.3.md new file mode 100644 index 000000000..bf0a9eb2e --- /dev/null +++ b/doc/changes/compositor/mr.841.3.md @@ -0,0 +1 @@ +main: Also resize on VK_SUBOPTIMAL_KHR. diff --git a/doc/changes/compositor/mr.864.md b/doc/changes/compositor/mr.864.md new file mode 100644 index 000000000..4297f5767 --- /dev/null +++ b/doc/changes/compositor/mr.864.md @@ -0,0 +1,2 @@ +multi: Make sure there are at least some predicted data, to avoid asserts in +non-service mode. diff --git a/doc/changes/compositor/mr.873.1.md b/doc/changes/compositor/mr.873.1.md new file mode 100644 index 000000000..63d234183 --- /dev/null +++ b/doc/changes/compositor/mr.873.1.md @@ -0,0 +1 @@ +main: Add new compute rendering backend. diff --git a/doc/changes/compositor/mr.873.2.md b/doc/changes/compositor/mr.873.2.md new file mode 100644 index 000000000..4e36721e4 --- /dev/null +++ b/doc/changes/compositor/mr.873.2.md @@ -0,0 +1 @@ +render: Add ATW support in the compute pipeline. diff --git a/doc/changes/drivers/mr.674.md b/doc/changes/drivers/mr.674.md new file mode 100644 index 000000000..93b361979 --- /dev/null +++ b/doc/changes/drivers/mr.674.md @@ -0,0 +1 @@ +vive: Factor out json config parser and reuse it in survive driver. diff --git a/doc/changes/drivers/mr.691.md b/doc/changes/drivers/mr.691.md new file mode 100644 index 000000000..db714093b --- /dev/null +++ b/doc/changes/drivers/mr.691.md @@ -0,0 +1 @@ +vive: Add rotation pose prediction to HMD and Controllers diff --git a/doc/changes/drivers/mr.715.md b/doc/changes/drivers/mr.715.md new file mode 100644 index 000000000..1fa0d2b33 --- /dev/null +++ b/doc/changes/drivers/mr.715.md @@ -0,0 +1 @@ +vf: Show the time on the video test source video server. diff --git a/doc/changes/drivers/mr.717.md b/doc/changes/drivers/mr.717.md new file mode 100644 index 000000000..7218fc44c --- /dev/null +++ b/doc/changes/drivers/mr.717.md @@ -0,0 +1,2 @@ +psvr: Ensure that timestamps are always after each other, stopping any +time-traveling sample packets. diff --git a/doc/changes/drivers/mr.740.md b/doc/changes/drivers/mr.740.md new file mode 100644 index 000000000..9895eac1c --- /dev/null +++ b/doc/changes/drivers/mr.740.md @@ -0,0 +1 @@ +vive: Setup the variable tracking for imu fusion. diff --git a/doc/changes/drivers/mr.741.md b/doc/changes/drivers/mr.741.md new file mode 100644 index 000000000..40a298770 --- /dev/null +++ b/doc/changes/drivers/mr.741.md @@ -0,0 +1 @@ +multi: Enable specifying arbitrary xrt_input_name for querying tracker poses. diff --git a/doc/changes/drivers/mr.742.md b/doc/changes/drivers/mr.742.md new file mode 100644 index 000000000..2e30da13c --- /dev/null +++ b/doc/changes/drivers/mr.742.md @@ -0,0 +1 @@ +ohmd: Support OpenHMD controllers and specifically the Oculus Touch controller. diff --git a/doc/changes/drivers/mr.774.md b/doc/changes/drivers/mr.774.md new file mode 100644 index 000000000..c55e11d19 --- /dev/null +++ b/doc/changes/drivers/mr.774.md @@ -0,0 +1,4 @@ +--- +- mr.803 +--- +wmr: Initial commit of driver, 3DoF only. diff --git a/doc/changes/drivers/mr.836.md b/doc/changes/drivers/mr.836.md new file mode 100644 index 000000000..a33b2c6c9 --- /dev/null +++ b/doc/changes/drivers/mr.836.md @@ -0,0 +1,5 @@ +--- +- mr.831 +- mr.837 +--- +depthai: Add a new frameserver driver that uses supports the DepthAI cameras. diff --git a/doc/changes/drivers/mr.839.md b/doc/changes/drivers/mr.839.md new file mode 100644 index 000000000..dba1496e6 --- /dev/null +++ b/doc/changes/drivers/mr.839.md @@ -0,0 +1,5 @@ +north_star: +- Upstreams Moses Turner's "VIPD" distortion. +- Fixes the FOV calc on the v1/3D distortion. +- General improvement of code organization. +- Improved JSON parsing. diff --git a/doc/changes/drivers/mr.858.1.md b/doc/changes/drivers/mr.858.1.md new file mode 100644 index 000000000..db56e3510 --- /dev/null +++ b/doc/changes/drivers/mr.858.1.md @@ -0,0 +1 @@ +v4l2: Add tracing support. diff --git a/doc/changes/drivers/mr.858.2.md b/doc/changes/drivers/mr.858.2.md new file mode 100644 index 000000000..b7c28305c --- /dev/null +++ b/doc/changes/drivers/mr.858.2.md @@ -0,0 +1 @@ +depthai: Add tracing support. diff --git a/doc/changes/drivers/mr.860.md b/doc/changes/drivers/mr.860.md new file mode 100644 index 000000000..e652bac0a --- /dev/null +++ b/doc/changes/drivers/mr.860.md @@ -0,0 +1 @@ +vf: Some tidy, frame fixes and tracing support. diff --git a/doc/changes/ipc/mr.565.md b/doc/changes/ipc/mr.565.md new file mode 100644 index 000000000..5604ca24b --- /dev/null +++ b/doc/changes/ipc/mr.565.md @@ -0,0 +1,4 @@ +ipc: Use libbsd pidfile to detect running Monado instances. +Enables automatically deleting stale socket files. +The socket file is now placed in $XDG_RUNTIME_DIR/monado_comp_ipc by default +or falls back to /tmp/monado_comp_ipc again if $XDG_RUNTIME_DIR is not set. diff --git a/doc/changes/ipc/mr.685.md b/doc/changes/ipc/mr.685.md new file mode 100644 index 000000000..c950d0c65 --- /dev/null +++ b/doc/changes/ipc/mr.685.md @@ -0,0 +1 @@ +Factor out the IPC server mainloop into a per-platform structure. diff --git a/doc/changes/ipc/mr.694.md b/doc/changes/ipc/mr.694.md new file mode 100644 index 000000000..6f6e59d7b --- /dev/null +++ b/doc/changes/ipc/mr.694.md @@ -0,0 +1 @@ +all: Transfer HMD blend mode, don't drop it on the floor. diff --git a/doc/changes/ipc/mr.712.md b/doc/changes/ipc/mr.712.md new file mode 100644 index 000000000..51ddc6b13 --- /dev/null +++ b/doc/changes/ipc/mr.712.md @@ -0,0 +1 @@ +Support systemd socket activation with meson too. diff --git a/doc/changes/ipc/mr.721.md b/doc/changes/ipc/mr.721.md new file mode 100644 index 000000000..f8827c87d --- /dev/null +++ b/doc/changes/ipc/mr.721.md @@ -0,0 +1,8 @@ +--- +- mr.754 +- mr.768 +- mr.800 +- mr.846 +--- +Now that there is a interface that allows the compositor to support +multi-client rendering use that instead of doing our own rendering. diff --git a/doc/changes/ipc/mr.768.md b/doc/changes/ipc/mr.768.md new file mode 100644 index 000000000..fa529bcb3 --- /dev/null +++ b/doc/changes/ipc/mr.768.md @@ -0,0 +1,2 @@ +Ensure that functions that require the compositor can't be called if a session +has not been created yet. diff --git a/doc/changes/misc_features/mr.676.md b/doc/changes/misc_features/mr.676.md new file mode 100644 index 000000000..db28b3529 --- /dev/null +++ b/doc/changes/misc_features/mr.676.md @@ -0,0 +1,9 @@ +--- +- mr.703 +- mr.783 +- mr.808 +- mr.820 +- mr.817 +- mr.918 +--- +More improvements to the Android port. diff --git a/doc/changes/misc_features/mr.692.md b/doc/changes/misc_features/mr.692.md new file mode 100644 index 000000000..ebc019afa --- /dev/null +++ b/doc/changes/misc_features/mr.692.md @@ -0,0 +1 @@ +imgui: Add ImPlot demo window. diff --git a/doc/changes/misc_features/mr.695.md b/doc/changes/misc_features/mr.695.md new file mode 100644 index 000000000..ca3847532 --- /dev/null +++ b/doc/changes/misc_features/mr.695.md @@ -0,0 +1 @@ +Implement tracking overrides via wrapper devices and add a tracking override configuration gui. diff --git a/doc/changes/misc_features/mr.697.1.md b/doc/changes/misc_features/mr.697.1.md new file mode 100644 index 000000000..325c391c8 --- /dev/null +++ b/doc/changes/misc_features/mr.697.1.md @@ -0,0 +1,3 @@ +util: Add trace marker support code, this code uses the Linux trace_marker +kernel support to enable Monado to trace both function calls and other async +events. diff --git a/doc/changes/misc_features/mr.697.2.md b/doc/changes/misc_features/mr.697.2.md new file mode 100644 index 000000000..18ecdc09b --- /dev/null +++ b/doc/changes/misc_features/mr.697.2.md @@ -0,0 +1,2 @@ +util: Add frame timing helper code designed to use Vulkan display timing +extensions to get proper frame timing in the compositor. diff --git a/doc/changes/misc_features/mr.705.md b/doc/changes/misc_features/mr.705.md new file mode 100644 index 000000000..91657da3d --- /dev/null +++ b/doc/changes/misc_features/mr.705.md @@ -0,0 +1 @@ +external/glad: Add EGL extension EGL_KHR_no_config_context. diff --git a/doc/changes/misc_features/mr.715.md b/doc/changes/misc_features/mr.715.md new file mode 100644 index 000000000..c8facc4e1 --- /dev/null +++ b/doc/changes/misc_features/mr.715.md @@ -0,0 +1,3 @@ +a/gst: Add a small and fairly dumb framework for integrating gstreamer pipelines +into Monado pipelines. Enough to be able to push frames into it and use various +encoder elements. diff --git a/doc/changes/misc_features/mr.739.md b/doc/changes/misc_features/mr.739.md new file mode 100644 index 000000000..92ecfbc65 --- /dev/null +++ b/doc/changes/misc_features/mr.739.md @@ -0,0 +1,4 @@ +--- +- mr.743 +--- +More work on the Windows port: fix timing, waiting, sleeping, handling the message queue. diff --git a/doc/changes/misc_features/mr.785.md b/doc/changes/misc_features/mr.785.md new file mode 100644 index 000000000..c7fae496b --- /dev/null +++ b/doc/changes/misc_features/mr.785.md @@ -0,0 +1,4 @@ +--- +- issue.82 +--- +Add JSON Schema for config files. diff --git a/doc/changes/misc_features/mr.809.md b/doc/changes/misc_features/mr.809.md new file mode 100644 index 000000000..3d8819b12 --- /dev/null +++ b/doc/changes/misc_features/mr.809.md @@ -0,0 +1 @@ +For code that is implemented in C++, note that the default standard mode is now C++17 across all platforms and modules, instead of a mix of 14 and 17 like before. The CI remains the decider of what functionality is available, as it contains the oldest distribution we support (Debian Buster). diff --git a/doc/changes/misc_features/mr.858.1.md b/doc/changes/misc_features/mr.858.1.md new file mode 100644 index 000000000..e5703a2d4 --- /dev/null +++ b/doc/changes/misc_features/mr.858.1.md @@ -0,0 +1 @@ +u/trace_marker: Add sink categories. diff --git a/doc/changes/misc_features/mr.858.2.md b/doc/changes/misc_features/mr.858.2.md new file mode 100644 index 000000000..f5929ce92 --- /dev/null +++ b/doc/changes/misc_features/mr.858.2.md @@ -0,0 +1 @@ +u/sink: Add tracing support to sink functions. diff --git a/doc/changes/misc_features/mr.858.3.md b/doc/changes/misc_features/mr.858.3.md new file mode 100644 index 000000000..e691cf7a0 --- /dev/null +++ b/doc/changes/misc_features/mr.858.3.md @@ -0,0 +1 @@ +t/hsv: Add tracing support for timing info. diff --git a/doc/changes/misc_fixes/mr.735.md b/doc/changes/misc_fixes/mr.735.md new file mode 100644 index 000000000..be622ac5b --- /dev/null +++ b/doc/changes/misc_fixes/mr.735.md @@ -0,0 +1,2 @@ +logging: Fix the first message always getting printed due to un-initialized +variable. diff --git a/doc/changes/misc_fixes/mr.737.md b/doc/changes/misc_fixes/mr.737.md new file mode 100644 index 000000000..6c8ff1590 --- /dev/null +++ b/doc/changes/misc_fixes/mr.737.md @@ -0,0 +1 @@ +Ensure we are always initializing our mutexes. diff --git a/doc/changes/misc_fixes/mr.785.md b/doc/changes/misc_fixes/mr.785.md new file mode 100644 index 000000000..0612d7129 --- /dev/null +++ b/doc/changes/misc_fixes/mr.785.md @@ -0,0 +1 @@ +Make config file reading more robust. diff --git a/doc/changes/state_trackers/mr.686.md b/doc/changes/state_trackers/mr.686.md new file mode 100644 index 000000000..f3f434ec5 --- /dev/null +++ b/doc/changes/state_trackers/mr.686.md @@ -0,0 +1,2 @@ +prober: Minor fixes & tidy commits. Mostly around doc-comments and the string +descriptor getter function. diff --git a/doc/changes/state_trackers/mr.688.md b/doc/changes/state_trackers/mr.688.md new file mode 100644 index 000000000..e5a85b8b5 --- /dev/null +++ b/doc/changes/state_trackers/mr.688.md @@ -0,0 +1,2 @@ +OpenXR: Ignore XrSystemHandTrackingPropertiesEXT when hand tracking extension +is not enabled. diff --git a/doc/changes/state_trackers/mr.689.md b/doc/changes/state_trackers/mr.689.md new file mode 100644 index 000000000..04302c20b --- /dev/null +++ b/doc/changes/state_trackers/mr.689.md @@ -0,0 +1,8 @@ +--- +- mr.689 +- mr.690 +- mr.740 +--- +OpenXR: ~~Add quirk for UnrealEngine4.27 to disable depth/stencil buffer to work +around a bug where Unreal would forget to call acquire before wait image.~~ +This has been fixed in UnrealEngine and is no longer needed. diff --git a/doc/changes/state_trackers/mr.705.md b/doc/changes/state_trackers/mr.705.md new file mode 100644 index 000000000..f4a053c24 --- /dev/null +++ b/doc/changes/state_trackers/mr.705.md @@ -0,0 +1,2 @@ +OpenXR: Support EGL clients sending in no EGLConfig if the EGLDisplay supports +EGL_KHR_no_config_context. diff --git a/doc/changes/state_trackers/mr.715.md b/doc/changes/state_trackers/mr.715.md new file mode 100644 index 000000000..cc4d44d11 --- /dev/null +++ b/doc/changes/state_trackers/mr.715.md @@ -0,0 +1 @@ +gui: Add a GUI for recording videos from the Valve Index. diff --git a/doc/changes/state_trackers/mr.735.md b/doc/changes/state_trackers/mr.735.md new file mode 100644 index 000000000..c5ee872dd --- /dev/null +++ b/doc/changes/state_trackers/mr.735.md @@ -0,0 +1,2 @@ +prober: Change the default logging level to info so that people can see what +drivers are disabled. diff --git a/doc/changes/state_trackers/mr.740.md b/doc/changes/state_trackers/mr.740.md new file mode 100644 index 000000000..6af700fe9 --- /dev/null +++ b/doc/changes/state_trackers/mr.740.md @@ -0,0 +1 @@ +OpenXR: Unreal Engine 4 depth buffer quirk no longer needed. diff --git a/doc/changes/state_trackers/mr.759.md b/doc/changes/state_trackers/mr.759.md new file mode 100644 index 000000000..99e93a37f --- /dev/null +++ b/doc/changes/state_trackers/mr.759.md @@ -0,0 +1,2 @@ +OpenXR: Use new multi compositor controls to set visibility and z_order if +available. This is needed for when we are not in service mode. diff --git a/doc/changes/state_trackers/mr.830.md b/doc/changes/state_trackers/mr.830.md new file mode 100644 index 000000000..e1d8e2326 --- /dev/null +++ b/doc/changes/state_trackers/mr.830.md @@ -0,0 +1 @@ +gui: Show git description in `monado-gui` window title. diff --git a/doc/changes/state_trackers/mr.847.1.md b/doc/changes/state_trackers/mr.847.1.md new file mode 100644 index 000000000..3be52e3ee --- /dev/null +++ b/doc/changes/state_trackers/mr.847.1.md @@ -0,0 +1 @@ +OpenXR: Add prefix to gfx related session functions to improve sorting. diff --git a/doc/changes/state_trackers/mr.847.2.md b/doc/changes/state_trackers/mr.847.2.md new file mode 100644 index 000000000..75bb195c0 --- /dev/null +++ b/doc/changes/state_trackers/mr.847.2.md @@ -0,0 +1 @@ +OpenXR: Break out end frame handling to it's own file since it's so big. diff --git a/doc/changes/state_trackers/mr.847.3.md b/doc/changes/state_trackers/mr.847.3.md new file mode 100644 index 000000000..862bd899c --- /dev/null +++ b/doc/changes/state_trackers/mr.847.3.md @@ -0,0 +1 @@ +OpenXR: Fill in normalised sub-image offsets and sizes. diff --git a/doc/changes/state_trackers/mr.858.4.md b/doc/changes/state_trackers/mr.858.4.md new file mode 100644 index 000000000..159cbf26b --- /dev/null +++ b/doc/changes/state_trackers/mr.858.4.md @@ -0,0 +1 @@ +gui: Add tracing support diff --git a/doc/changes/state_trackers/mr.859.1.md b/doc/changes/state_trackers/mr.859.1.md new file mode 100644 index 000000000..0a2eb86de --- /dev/null +++ b/doc/changes/state_trackers/mr.859.1.md @@ -0,0 +1 @@ +gui: Various fixes for video handling, null checking and wrong argument orders. diff --git a/doc/changes/state_trackers/mr.859.2.md b/doc/changes/state_trackers/mr.859.2.md new file mode 100644 index 000000000..2c121b8a9 --- /dev/null +++ b/doc/changes/state_trackers/mr.859.2.md @@ -0,0 +1,2 @@ +gui: Add support to record from ELP 3D camera and select DepthAI camera +to calibration. diff --git a/doc/changes/state_trackers/mr.886.md b/doc/changes/state_trackers/mr.886.md new file mode 100644 index 000000000..abdcfc748 --- /dev/null +++ b/doc/changes/state_trackers/mr.886.md @@ -0,0 +1 @@ +OpenXR: Add support for XR_KHR_swapchain_usage_input_attachment_bit. diff --git a/doc/changes/xrt/mr.697.md b/doc/changes/xrt/mr.697.md new file mode 100644 index 000000000..c8da2b955 --- /dev/null +++ b/doc/changes/xrt/mr.697.md @@ -0,0 +1,4 @@ +Added frame timing code that when the underlying vulkan driver supports the +VK_GOOGLE_display_timing extension greatly improves the timing accerecy of the +compositor. Along with this tracing code was added to better help use understand +what was happening during a frame. diff --git a/doc/changes/xrt/mr.704.md b/doc/changes/xrt/mr.704.md new file mode 100644 index 000000000..39a12eb6b --- /dev/null +++ b/doc/changes/xrt/mr.704.md @@ -0,0 +1 @@ +xrt: Add functionality to disable individual drivers in the configuration file. diff --git a/doc/changes/xrt/mr.705.1.md b/doc/changes/xrt/mr.705.1.md new file mode 100644 index 000000000..908280e8e --- /dev/null +++ b/doc/changes/xrt/mr.705.1.md @@ -0,0 +1 @@ +xrt: Return xrt_result_t from xrt_gfx_provider_create_gl_egl diff --git a/doc/changes/xrt/mr.705.2.md b/doc/changes/xrt/mr.705.2.md new file mode 100644 index 000000000..9e17e3485 --- /dev/null +++ b/doc/changes/xrt/mr.705.2.md @@ -0,0 +1,5 @@ +--- +- mr.768 +--- +xrt: Add XRT_ERROR_EGL_CONFIG_MISSING error, to handle missing config from +EGL compositor creation call. diff --git a/doc/changes/xrt/mr.715.md b/doc/changes/xrt/mr.715.md new file mode 100644 index 000000000..78fcc20f6 --- /dev/null +++ b/doc/changes/xrt/mr.715.md @@ -0,0 +1 @@ +Add small helper function for pushing frames. diff --git a/doc/changes/xrt/mr.721.1.md b/doc/changes/xrt/mr.721.1.md new file mode 100644 index 000000000..df57756ef --- /dev/null +++ b/doc/changes/xrt/mr.721.1.md @@ -0,0 +1,2 @@ +Add `xrt_compositor_fence` interface to handle service and client render +syncronisation. diff --git a/doc/changes/xrt/mr.721.2.md b/doc/changes/xrt/mr.721.2.md new file mode 100644 index 000000000..830d321ac --- /dev/null +++ b/doc/changes/xrt/mr.721.2.md @@ -0,0 +1 @@ +Add `XRT_ERROR_THREADING_INIT_FAILURE` a new threading related error code. diff --git a/doc/changes/xrt/mr.721.3.md b/doc/changes/xrt/mr.721.3.md new file mode 100644 index 000000000..3c5ec00e7 --- /dev/null +++ b/doc/changes/xrt/mr.721.3.md @@ -0,0 +1,3 @@ +Add alternative functions to `xrt_compositor::wait_frame` called +`xrt_compositor::predict_frame` and `xrt_compositor::mark_frame` these allow one +to do frame timing without having to block on the service side. diff --git a/doc/changes/xrt/mr.721.4.md b/doc/changes/xrt/mr.721.4.md new file mode 100644 index 000000000..b3811a302 --- /dev/null +++ b/doc/changes/xrt/mr.721.4.md @@ -0,0 +1,4 @@ +Add `xrt_multi_compositor_control` that allows the `xrt_system_compositor` to +expose a interface that the IPC layer can use to manage multiple clients +without having to do the rendering. This allows us to move a lot of the code +out the IPC layer and make it more about just passing calls. diff --git a/doc/changes/xrt/mr.721.5.md b/doc/changes/xrt/mr.721.5.md new file mode 100644 index 000000000..f9e3088ac --- /dev/null +++ b/doc/changes/xrt/mr.721.5.md @@ -0,0 +1,3 @@ +Pass `XrFrameEndInfo::displayTime` to `xrt_compositor::layer_begin` so that the +compositor can correctly schedule frames, most importantly do not display them +too early that might lead to stutter. diff --git a/doc/changes/xrt/mr.723.md b/doc/changes/xrt/mr.723.md new file mode 100644 index 000000000..5b03d6c5f --- /dev/null +++ b/doc/changes/xrt/mr.723.md @@ -0,0 +1,7 @@ +--- +- mr.754 +- mr.807 +--- +Make @ref xrt_swapchain be reference counted. This will greatly help with +handling swapchains for multiple clients in the compositor rendering pipeline +where a client might go away while the compositor is using it. diff --git a/doc/changes/xrt/mr.749.md b/doc/changes/xrt/mr.749.md new file mode 100644 index 000000000..d761f5da1 --- /dev/null +++ b/doc/changes/xrt/mr.749.md @@ -0,0 +1 @@ +Make `enum xrt_blend_mode` an array instead of a bitfield, so that drivers can specify which one is preferred. \ No newline at end of file diff --git a/doc/changes/xrt/mr.768.md b/doc/changes/xrt/mr.768.md new file mode 100644 index 000000000..7e2ba3dfe --- /dev/null +++ b/doc/changes/xrt/mr.768.md @@ -0,0 +1,2 @@ +Add new IPC session not created error `XRT_ERROR_IPC_SESSION_NOT_CREATED`, as +some functions can not be called without first creating a session. diff --git a/doc/changes/xrt/mr.794.md b/doc/changes/xrt/mr.794.md new file mode 100644 index 000000000..891e7dda6 --- /dev/null +++ b/doc/changes/xrt/mr.794.md @@ -0,0 +1,2 @@ +Make eye_relation argument to xrt_device_get_view_pose const, more safety for +everybody. diff --git a/doc/changes/xrt/mr.800.md b/doc/changes/xrt/mr.800.md new file mode 100644 index 000000000..a7a3e1f03 --- /dev/null +++ b/doc/changes/xrt/mr.800.md @@ -0,0 +1,2 @@ +Add XRT_ERROR_IPC_SESSION_ALREADY_CREATED error message to signal that a session +has already been created on this connection. diff --git a/doc/changes/xrt/mr.810.md b/doc/changes/xrt/mr.810.md new file mode 100644 index 000000000..2bef57966 --- /dev/null +++ b/doc/changes/xrt/mr.810.md @@ -0,0 +1,4 @@ +--- +- issue.61 +--- +Add a @ref conventions page. diff --git a/doc/changes/xrt/mr.847.md b/doc/changes/xrt/mr.847.md new file mode 100644 index 000000000..6f43e0be3 --- /dev/null +++ b/doc/changes/xrt/mr.847.md @@ -0,0 +1,2 @@ +Send down sub-image offsets and sizes in normalised form, this makes it so that +compositor does not need to track the size of swapchains. diff --git a/doc/changes/xrt/mr.867.md b/doc/changes/xrt/mr.867.md new file mode 100644 index 000000000..5cb4c2757 --- /dev/null +++ b/doc/changes/xrt/mr.867.md @@ -0,0 +1,2 @@ +Add `use_dedicated_allocation` field to `xrt_image_native` struct to track if +dedicated allocation is required. diff --git a/doc/changes/xrt/mr.870.md b/doc/changes/xrt/mr.870.md new file mode 100644 index 000000000..773591e2b --- /dev/null +++ b/doc/changes/xrt/mr.870.md @@ -0,0 +1 @@ +Add xrt_vec3_f64 struct. diff --git a/doc/conventions.md b/doc/conventions.md new file mode 100644 index 000000000..3ab47bad3 --- /dev/null +++ b/doc/conventions.md @@ -0,0 +1,165 @@ +# Code Style and Conventions {#conventions} + + + + + +Here are some general code style guidelines we follow. + +## APIs + +Internal APIs, when it makes sense, should be C APIs. Headers that define +general communication interfaces between modules (not just use of utilities) +belong in the `xrt/include/xrt` directory, and should not depend on any other module outside +that directory. (As a historical note: this directory gets its name from a +compressed version of the phrase "XR RunTime", a generic term for Monado and an +early development codename. Also, it's shorter than `monado_` and so nicer to +use in code.) + +What follows are some basic API usage rules. Note that all the module usage +relations must be expressed in the build system, so module usage should form a +directed-acyclic-graph. + +- Any module can implement or use APIs declared in `xrt/include/xrt` +- Any module (except the `xrt` interface headers themselves) can (and should!) + use APIs declared in `xrt/auxiliary/util`. +- Any module except for `auxiliary/util` and the `xrt` interface headers + themselves can use APIs declared in other `xrt/auxiliary` modules. + +## Naming + +- C APIs: + - `lower_snake_case` for types and functions. + - `UPPER_SNAKE_CASE` for macros. e.g. @ref U_TYPED_CALLOC (which is how all + allocations in C code should be performed) + - Prefix names with a "namespace" - the library/module where they reside. e.g. + @ref u_var_add_root, @ref math_pose_validate + - Related: only things prefixed by `xrt_` belong in the `xrt/include/xrt` + directory, and nothing named starting with `xrt_` should be declared + anywhere else. (Interfaces *declared* in `xrt/include/xrt` are + *implemented* in other modules.) + - Generally, we do not declare typedefs for `struct` and `enum` types, but + instead refer to them in long form, saying `struct` or `enum` then the name. + - If a typedef is needed, it should be named ending with `_t`. + - Parameters: `lower_snake_case` or acronyms. + - Output parameters should begin with `out_`. + - Of special note: Structures/types that represent "objects" often have long + type names or "conceptual" names. When a pointer to them is passed to a + function or kept as a local variable, it is typically named by taking the + first letter of each (typically `_`-delimited) word in the structure type + name. Sometimes, it is an abbreviated form of that name instead. Relevant + examples: + - @ref xrt_comp_native_create_swapchain() is a member function of the + interface @ref xrt_compositor_native, and takes a pointer to that + interface named `xcn`. It creates an @ref xrt_swapchain, which it + populates in the parameter named `out_xscn`: `out_` because it's a + purely output parameter, `xscn` from @ref xrt_swapchain_native + specifically the letters `Xrt_SwapChain_Native`. @ref xrt_swapchain and + related types are a small exception to the rules - there are only 2 + words if you go by the `_` delimiters, but for clarity we treat + swapchain as if it were two words when abbreviating. A few other places + in the `xrt` headers use `x` + an abbreviated name form, like `xinst` + for @ref xrt_instance, `xdev` for @ref xrt_device, `xsysc` sometimes + used for @ref xrt_system_compositor. + - `create` and `destroy` are used when the functions actually perform + allocation and return the new object, or deallocation of the passed-in + object. + - If some initialization or cleanup is required but the type is not opaque and + is allocated by the caller, the names to use are `init` and, if needed, one + of `cleanup`/`fini`/`teardown`. (We are not yet consistent on these names.) + One common example is when there is some shared code and a structure + partially implementing an interface: a further-derived object may need to + call an `init` function on the shared structure, but it was allocated by the + derived object and held by value. +- C++: + - Where a C API is exposed, it should follow the C API naming schemes. + - If only a C++ API is exposed, a fairly conventional C++ naming scheme is used: + - Namespaces: nested to match directory structure, starting with `xrt::`. + (Migration to this pattern is still in progress.) + - There are no C++ interfaces in the `xrt/include/xrt`, by design, so this + is not ambiguous. + - Place types that need to be exposed in a header for technical reasons, + but that are still considered implementation details, within a + further-nested `detail` namespace, as seen elsewhere in the C++ + ecosystem. + - Types/classes: `CamelCase` + - Methods/functions: `lowerCamelCase` + - If a header is only usable from C++ code, it should be named with the + extension `.hpp` to signify this. + +## Patterns and Idioms + +This is an incomplete list of conventional idioms used in the Monado codebase. + +### C "Inheritance" through first struct member + +Despite being in C, the design is fairly object-oriented. Types implement +interfaces and derive from other types typically by placing a field of that +parent type/interface as their first element, conventionally named `base`. This +means that a pointer to the derived type, and a pointer to the base type, have +the same value. + +For example, consider @ref client_gl_swapchain + +- Its first element is named @ref client_gl_swapchain::base and is of type @ref + xrt_swapchain_gl - meaning that it implements @ref xrt_swapchain_gl +- @ref xrt_swapchain_gl in turn starts with @ref xrt_swapchain_gl::base which is + @ref xrt_swapchain - meaning that @ref xrt_swapchain_gl **extends** @ref + xrt_swapchain. (Both @ref xrt_swapchain_gl and @ref xrt_swapchain are abstract + interfaces, as indicated by the `xrt_` prefix.) + +Structures/types that represent "objects" are often passed as the first +parameter to many functions, which serve as their "member functions". Sometimes, +these types are opaque and not related to other types in the system in a +user-visible way: they should have a `_create` and `_destroy` function. See @ref +time_state, @ref time_state_create, @ref time_state_destroy + +In other cases, an interface will have function pointers defined as fields in +the interface structure. (A type implementing these may be opaque, but would +begin with a member of the interface/base type.) These interface function +pointers must still take in a self pointer as their first parameter, because +there is no implied `this` pointer in C. This would result in awkward calls with +repeated, error-prone mentions of the object pointer, such as this example +calling the @ref xrt_device::update_inputs interface: +`xdev->update_inputs(xdev)`. These are typically wrapped by inline free +functions that make the call through the function pointer. Considering again the +@ref xrt_device example, the way you would call @ref xrt_device::update_inputs +is actually @ref xrt_device_update_inputs(). + +### Destroy takes a pointer to a pointer, nulls it out + +Destroy free functions should take a pointer to a pointer, performing null checks +before destruction, and setting null. They always succeed (void return): a +failure when destroying an object has little meaning in most cases. For a +sample, see @ref xrt_images_destroy. It would be used like this: + +```c +struct xrt_image_native_allocator *xina = /* created and initialized, or maybe NULL */; + +/* ... */ + +xrt_images_destroy(&xina); + +/* here, xina is NULL in all cases, and if it wasn't NULL before, it has been freed. */ +``` + +Note that this pattern is used in most cases but not all in the codebase: we +are gradually migrating those that don't fit this pattern. If you call a +destroy function that does not take a pointer-to-a-pointer, make sure to do +null checks before calling and set it to null after it returns. + +Also note: when an interface includes a "destroy" function pointer, it just +takes the normal pointer to an object: The free function wrapper is the one that +takes a pointer-to-a-pointer and handles the null checks. See for example @ref +xrt_instance_destroy takes the pointer-to-a-pointer, while the interface method +@ref xrt_instance::destroy takes just the single pointer. diff --git a/doc/example_configs/config_v0.northstar_lonestar.json b/doc/example_configs/config_v0.northstar_lonestar.json new file mode 100644 index 000000000..2f4ece802 --- /dev/null +++ b/doc/example_configs/config_v0.northstar_lonestar.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://monado.pages.freedesktop.org/monado/config_v0.schema.json", + "active": "tracking", + "tracking": { + "tracking_overrides": [ + { + "target_device_serial": "North Star", + "tracker_device_serial": "Intel RealSense Device-SLAM", + "type": "direct", + "offset": { + "orientation": { + "x": -0.102931, + "y": 0, + "z": 0, + "w": 0.994689 + }, + "position": { + "x": 0, + "y": 0.0683, + "z": -0.0744 + } + }, + "xrt_input_name": "XRT_INPUT_GENERIC_TRACKER_POSE" + }, + { + "target_device_serial": "Leap Motion v2 driver", + "tracker_device_serial": "Intel RealSense Device-SLAM", + "type": "attached", + "offset": { + "orientation": { + "x": 0, + "y": 0, + "z": 0, + "w": 1 + }, + "position": { + "x": 0, + "y": 0.005, + "z": 0 + } + }, + "xrt_input_name": "XRT_INPUT_GENERIC_TRACKER_POSE" + } + ], + "version": 0 + } +} diff --git a/doc/example_configs/config_v0.northstar_lonestar.json.license b/doc/example_configs/config_v0.northstar_lonestar.json.license new file mode 100644 index 000000000..c7f54ea2c --- /dev/null +++ b/doc/example_configs/config_v0.northstar_lonestar.json.license @@ -0,0 +1,3 @@ +Copyright 2021, Moses Turner +Copyright 2021, Collabora, Ltd. +SPDX-License-Identifier: CC0-1.0 diff --git a/doc/example_configs/config_v0.schema.json b/doc/example_configs/config_v0.schema.json new file mode 100644 index 000000000..d066635f3 --- /dev/null +++ b/doc/example_configs/config_v0.schema.json @@ -0,0 +1,228 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "title": "Monado configuration file schema v0", + "type": "object", + "properties": { + "$schema": { + "type": "string", + "title": "JSON Schema directive", + "default": "https://monado.pages.freedesktop.org/monado/config_v0.schema.json" + }, + "active": { + "type": "string", + "title": "Name of the active config object", + "description": "Must match a key in the top-level object. Only config data in that object will be used.", + "$comment": "have omitted 'none' because it primarily exists for environment overrides: just leave out this key in the json for 'none'", + "enum": [ + "tracking", + "remote" + ] + }, + "tracking": { + "$ref": "#/definitions/tracking" + }, + "remote": { + "$ref": "#/definitions/remote" + } + }, + "definitions": { + "remote": { + "title": "Remote device configuration", + "required": [ + "version" + ], + "properties": { + "port": { + "type": "integer" + }, + "version": { + "type": "integer", + "title": "Remote config file schema version", + "enum": [ + 0 + ] + } + } + }, + "tracking": { + "title": "Tracking configuration", + "required": [ + "version" + ], + "properties": { + "version": { + "type": "integer", + "title": "Tracking config file schema version", + "enum": [ + 0 + ] + }, + "tracking_overrides": { + "$ref": "#/definitions/tracking_overrides" + }, + "camera_name": { + "type": "string" + }, + "camera_mode": { + "type": "integer" + }, + "camera_type": { + "type": "string", + "enum": [ + "regular_mono", + "regular_sbs", + "ps4", + "leap_motion" + ] + }, + "calibration_path": { + "type": "string" + }, + } + }, + "tracking_overrides": { + "title": "Tracking overrides", + "type": "array", + "items": { + "$ref": "#/definitions/tracking_override" + } + }, + "tracking_override": { + "type": "object", + "title": "Tracking override object", + "required": [ + "target_device_serial", + "tracker_device_serial", + "type" + ], + "properties": { + "target_device_serial": { + "title": "Target Device Serial", + "$ref": "#/definitions/device_serial" + }, + "tracker_device_serial": { + "title": "Tracker Device Serial", + "$ref": "#/definitions/device_serial" + }, + "type": { + "type": "string", + "enum": [ + "direct", + "attached" + ] + }, + "offset": { + "type": "object", + "title": "Tracking offset", + "default": { + "orientation": { + "x": 0, + "y": 0, + "z": 0, + "w": 1 + }, + "position": { + "x": 0, + "y": 0, + "z": 0 + } + }, + "required": [ + "orientation", + "position" + ], + "properties": { + "orientation": { + "$ref": "#/definitions/quaternion" + }, + "position": { + "title": "Position offset", + "$ref": "#/definitions/vector3" + } + } + }, + "xrt_input_name": { + "type": "string", + "enum": [ + "XRT_INPUT_HYDRA_POSE", + "XRT_INPUT_TOUCH_AIM_POSE", + "XRT_INPUT_SIMPLE_AIM_POSE", + "XRT_INPUT_GENERIC_HEAD_DETECT", + "XRT_INPUT_SIMPLE_GRIP_POSE", + "XRT_INPUT_INDEX_GRIP_POSE", + "XRT_INPUT_VIVE_GRIP_POSE", + "XRT_INPUT_PSMV_AIM_POSE", + "XRT_INPUT_WMR_AIM_POSE", + "XRT_INPUT_WMR_GRIP_POSE", + "XRT_INPUT_PSMV_BODY_CENTER_POSE", + "XRT_INPUT_HAND_AIM_POSE", + "XRT_INPUT_INDEX_AIM_POSE", + "XRT_INPUT_DAYDREAM_POSE", + "XRT_INPUT_GENERIC_HEAD_POSE", + "XRT_INPUT_PSMV_BALL_CENTER_POSE", + "XRT_INPUT_GO_GRIP_POSE", + "XRT_INPUT_TOUCH_GRIP_POSE", + "XRT_INPUT_HAND_GRIP_POSE", + "XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT", + "XRT_INPUT_GENERIC_HAND_TRACKING_LEFT", + "XRT_INPUT_GENERIC_TRACKER_POSE", + "XRT_INPUT_PSMV_GRIP_POSE", + "XRT_INPUT_GO_AIM_POSE", + "XRT_INPUT_VIVE_AIM_POSE" + ], + "default": "XRT_INPUT_GENERIC_TRACKER_POSE" + } + } + }, + "device_serial": { + "type": "string", + "title": "Device serial", + "description": "The unique string assigned to a particular instance of a device by the driver." + }, + "quaternion": { + "type": "object", + "title": "Quaternion", + "description": "Should be normalized (length == 1) to represent rotation", + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + }, + "z": { + "type": "number" + }, + "w": { + "type": "number" + } + }, + "required": [ + "x", + "y", + "z", + "w" + ] + }, + "vec3": { + "type": "object", + "title": "Vector3", + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + }, + "z": { + "type": "number" + } + }, + "required": [ + "x", + "y", + "z" + ] + } + } +} diff --git a/doc/example_configs/config_v0.schema.json.license b/doc/example_configs/config_v0.schema.json.license new file mode 100644 index 000000000..a1d1e58cc --- /dev/null +++ b/doc/example_configs/config_v0.schema.json.license @@ -0,0 +1,2 @@ +Copyright 2021, Collabora, Ltd. +SPDX-License-Identifier: CC0-1.0 diff --git a/doc/frame-timing.md b/doc/frame-timing.md new file mode 100644 index 000000000..30026fb3c --- /dev/null +++ b/doc/frame-timing.md @@ -0,0 +1,48 @@ +# Frame Timing {#frame-timing} + + + +A "brief" overview of the various time-points that a frame goes through, from +when the application gets go ahead to render the frame to when pixels are turned +into photons. This only a single frame, where all of the timings are hit and the +application is single threaded. The HMD also only turns on the display during +the vblank period, meaning the pixel to photon transformation is delayed from +scanout starting to the vblank period (like for the Index). + +* `xrWaitFrame` returns to the application, referred to as **wake_up**. +* The app does a logic step to move the simulation to the next predicted + display time. +* `xrBeginFrame` is called by the application, referred to as **begin**. +* The app renders the current views using the GPU. +* `xrEndFrame` is called by the application submitting the views. +* The compositor wakes up to do it's distorting rendering and any warping, + checking if the applications rendering has finished. When the compositor + submits the work to the GPU, referred to as **submit**. +* The compositor queues it's swapbuffer to the display engine. +* Scanout starts, the kernel checked if the compositors rendering was completed + in time. We refer to this time as **present**, this seems to be common. +* During the vblank period the display lights up turning the pixels into + photons. We refer to this time as **display**, same as in OpenXR. + +The names for timepoints are chosen to align with the naming in +[`VK_GOOGLE_display_timing`][], reading that extension can provide further +information. + +## Main compositor perspective + +* @ref xrt_comp_wait_frame - It is within this function that the frame timing is + predicted, see @ref u_rt_predict. The compositor will then wait to + **wake_up** time and then return from this function. +* @ref xrt_comp_begin_frame - The app or IPC server calls this function when it + is done with CPU work and ready to do GPU work. +* @ref xrt_comp_discard_frame - The frame is discarded. +* @ref xrt_comp_layer_begin - Called during transfers of layers. +* @ref xrt_comp_layer_stereo_projection - This and other layer functions are + called to list the layers the compositor should render. +* @ref xrt_comp_layer_commit - The compositor starts to render the frame, + trying to hit the **present** time. + +[`VK_GOOGLE_display_timing`]: https://www.khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VK_GOOGLE_display_timing.html diff --git a/doc/howto-release.md b/doc/howto-release.md index 828b004e9..4a8fbe6fa 100644 --- a/doc/howto-release.md +++ b/doc/howto-release.md @@ -1,4 +1,9 @@ -# How to release +# How to make a release {#how-to-release} + + These instructions assumes that the version you are making is `21.0.0`. @@ -17,7 +22,6 @@ git commit -m"doc: Update CHANGELOG.md" doc/CHANGELOG.md git commit -m"doc: Remove old changelog fragments" doc/changes ``` - ## Update versions Edit the files @@ -32,7 +36,6 @@ See previous commits for exact places. git commit -a -m"monado: Update version" ``` - ## Tag the code Do the tagging from git, do **not** do it from gitlab, also make sure to prefix @@ -42,7 +45,6 @@ the version with `v` so that `21.0.0` becomes `v21.0.0`. git tag v21.0.0 -m"v21.0.0" ``` - ## Do gitlab release The Gitlab UI has a friendly interface, just follow the guide there. diff --git a/doc/implementing-extensions.md b/doc/implementing-extensions.md new file mode 100644 index 000000000..3e33ed848 --- /dev/null +++ b/doc/implementing-extensions.md @@ -0,0 +1,31 @@ +# Implementing OpenXR extensions {#implementing-extension} + + + +Khronos often adds new functionality to the OpenXR specification as extensions. + +The general steps to implement an OpenXR extension in Monado are as follows. + +* Edit scripts/generate_oxr_ext_support.py. Usually you only need to add an + entry to the `EXTENSIONS` list at the top. +* Run the script `python scripts/generate_oxr_ext_support.py`. +* Format the regenerated file with + `clang-format -i src/xrt/state_trackers/oxr/oxr_extension_support.h`. +* Add entry points for each new function in + `src/xrt/state_trackers/oxr/oxr_api_negotiate.c`. +* Monado internal implementations of "objects" (think XrSession or + XrHandTracker) go into `src/xrt/state_trackers/oxr/oxr_objects.h`. +* Enums, defines and types go into `src/xrt/include/xrt/xrt_defines.h`. OpenXR + types are not used outside of the `oxr_api_*` files, instead equivalents with + the prefix `XRT_` are defined here. +* Add Monado specific prototypes for the new functions in + `src/xrt/state_trackers/oxr/oxr_objects.h`. The Monado implementations of + OpenXR functions are prefixed with `oxr_`. +* Implement the Monado specific functions in an appropriate source file + `src/state_trackers/oxr/oxr_api_*.c`. Trivial functions can be implemented + right there along with the usual parameter checks. More complex functions that + access more internal monado state should call functions implemented in the + relevant `oxr_*.c` file (without `_api_`). diff --git a/doc/ipc.md b/doc/ipc.md new file mode 100644 index 000000000..3dd547b78 --- /dev/null +++ b/doc/ipc.md @@ -0,0 +1,173 @@ +# IPC Design {#ipc-design} + + + +- Last updated: 24-February-2021 + +When the service starts, an `xrt_instance` is created and selected, a native +system compositor is initialized, a shared memory segment for device data is +initialized, and other internal state is set up. (See `ipc_server_process.c`.) + +There are three main communication needs: + +- The client shared library needs to be able to **locate** a running service, if + any, to start communication. (Auto-starting, where available, is handled by + platform-specific mechanisms: the client currently has no code to explicitly + start up the service.) +- The client and service must share a dedicated channel for IPC calls (aka + **RPC** - remote procedure call), typically a socket. +- The service must share access to device data updating at various rates, shared + by all clients. This is typically done with a form of **shared memory**. + +Each platform's implementation has a way of meeting each of these needs. The +specific way each need is met is highlighted below. + +## Linux Platform Details + +In an typical Linux environment, the Monado service can be launched one of two +ways: manually, or by socket activation (e.g. from systemd). In either case, +there is a Unix domain socket with a well-known name (known at compile time, and +built-in to both the service executable and the client shared library) used by +clients to connect to the service: this provides the **locating** function. +This socket is polled in the service mainloop, using epoll, to detect any new +client connections. + +Upon a client connection to this "locating" socket, the service will [accept][] +the connection, returning an FD, which is passed to +`start_client_listener_thread()` to start a thread specific to that client. The +FD produced this way is now also used for the IPC calls - the **RPC** function - +since it is specific to that client-server communication channel. One of the +first calls made transports a duplicate of the **shared memory** segment file +descriptor to the client, so it has (read) access to this data. + +[accept]: https://man7.org/linux/man-pages/man2/accept.2.html + +## Android Platform Details + +On Android, in order to pass platform objects, allow for service activation, and +fit better within the idioms of the platform, Monado provides a Binder/AIDL +service instead of a named socket. (The named sockets we typically use are not +permitted by the platform, and "abstract" named sockets are currently available, +but are not idiomatic for the platform and lack other useful capabilities.) +Specifically, we provide a [foreground and started][foreground] (to be able to +display), [bound][bound_service] [service][android_service] with an interface +defined using [AIDL][]. (See also +[this third-party guide about such AIDL services][AidlServices]) This is not +like the system services which provide hardware data or system framework data +from native code. this has a Java (JVM/Dalvik/ART) component provided by code in +an APK, exposed by properties in the package manifest. + +[NdkBinder][] is not used because it is mainly suitable for the system type of +binder services. An APK-based service would still require some JVM code to +expose it, and since the AIDL service is used for so little, mixing languages +did not make sense. + +The service we expose provides an implementation of our AIDL-described +interface, `org.freedesktop.monado.ipc.IMonado`. This can be modified freely, as +both the client and server are built at the same time and packaged in the same +APK, even though they get loaded in different processes. + +[foreground]: https://developer.android.com/guide/components/foreground-services +[bound_service]: https://developer.android.com/guide/components/bound-services +[android_service]: https://developer.android.com/guide/components/services +[aidl]: https://developer.android.com/guide/components/aidl +[AidlServices]: https://devarea.com/android-services-and-aidl/ +[NdkBinder]: https://developer.android.com/ndk/reference/group/ndk-binder + +The first main purpose of this service is for automatic startup and the +**locating** function: helping establish communication between the client and +the service. The Android framework takes care of launching the service process +when the client requests to start and bind our service by name and package. The +framework also provides us with method calls when we're started/bound. In this +way, the "entry point" of the Monado service on Android is the +`org.freedesktop.monado.ipc.MonadoService` class, which exposes the +implementation of our AIDL interface, `org.freedesktop.monado.ipc.MonadoImpl`. + +From there, the native-code mainloop starts when this service is started. By +default, the JVM code will signal the mainloop to shut down a short time after +the last client disconnects, to work best within the platform. + +At startup, just as on Linux, the shared memory segment is created. The +[ashmem][] API is used to create/destroy an anonymous **shared memory** segment +on Android, instead of standard POSIX shared memory, but is otherwise treated +and used exactly the same as on standard Linux: file descriptors are duplicated +and passed through IPC calls, etc. + +When the client side starts up, it creates an __anonymous socket pair__ to use +for IPC calls (the **RPC** function) later. It then passes one of the two file +descriptors into the AIDL method we defined named "connect". This transports the +FD to the service process, which uses it as the unique communication channel for +that client in its own thread. This replaces the socket pair produced by +connecting/accepting the named socket as used in standard Linux. + +[ashmem]: https://developer.android.com/ndk/reference/group/memory + +The AIDL interface is also used for transporting some platform objects. At this +time, the only one transported in this way is the [Surface][] injected into the +client activity which is used for displaying rendered output. + +[Surface]: https://developer.android.com/reference/android/view/Surface + +### Synchronization + +Synchronization of new client connections is a special challenge on the Android +platform, since new clients arrive via calls into JVM code while the mainloop is +native code. Unlike Linux, we cannot simply use epoll to check if there are new +connections to our locating socket. + +We have the following design goals/constraints: + +- All we need to communicate is an integer (file descriptor) within a process. +- Make it fast in the server mainloop in the most common case that there are no + new clients. + - This suggests that we should be able to check if there may be a waiting + client in purely native code, without JNI. +- Make it relatively fast in the server mainloop even when there is a client, + since it's the compositor thread. + - This might mean we want to do it all without JNI on the main thread. +- The client should know (and be unblocked) when the server has accepted its + connection. + - This suggests that the method called in `MonadoImpl` should block until the + server consumes/accepts the connection. + - Not 100% sure this is required, but maybe. +- Resources (file descriptors, etc) should not be leaked. + - Each should have a well-known owner at each point in time. +- It is OK if only one new client is accepted per mainloop. + - The mainloop is high rate (compositor rate) and new client connections are + relatively infrequent. + +The IPC service creates a pipe as well as some state variables, two mutexes, and a +condition variable. + +When the JVM Service code has a new client, it calls +`ipc_server_mainloop_add_fd()` to pass the FD in. It takes two mutexes, in +order: `ipc_server_mainloop::client_push_mutex` and +`ipc_server_mainloop::accept_mutex`. The purpose of +`ipc_server_mainloop::client_push_mutex` is to allow only one client into the +client-acceptance handshake at a time, so that no acknowledgement of client +accept is lost. Once those two mutexes are locked, +`ipc_server_mainloop_add_fd()` writes the FD number to the pipe. Then, it waits +on the condition variable (releasing `accept_mutex`) to see either that FD +number or the special "shutting down" sentinel value in the `last_accepted_fd` +variable. If it sees the FD number, that indicates that the other side of the +communication (the mainloop) has taken ownership of the FD and will handle +closing it. If it sees the sentinel value, or has an error at some point, it +assumes that ownership is retained and it should close the FD itself. + +The other side of the communication works as follows: epoll is used to check if +there is new data waiting on the pipe. If so, the +`ipc_server_mainloop::accept_mutex` lock is taken, and an FD number is read from +the pipe. A client thread is launched for that FD, then the `last_accepted_fd` +variable is updated and the `ipc_server_mainloop::accept_cond` condition +variable signalled. + +The initial plan required that the server also wait on +`ipc_server_mainloop::accept_cond` for the `last_accepted_fd` to be reset back +to `0` by the acknowledged client, thus preventing losing acknowledgements. +However, it is undesirable for the clients to be able to block the +compositor/server, so this wait was considered not acceptable. Instead, the +`ipc_server_mainloop::client_push_mutex` is used so that at most one +un-acknowledged client may have written to the pipe at any given time. diff --git a/doc/mainpage.md b/doc/mainpage.md index b92bf73c8..5e8626295 100644 --- a/doc/mainpage.md +++ b/doc/mainpage.md @@ -1,7 +1,7 @@ # Monado @@ -11,12 +11,17 @@ getting started information and general documentation. ## Useful pages -* @ref md_CHANGELOG - If this is the web version of the docs, the changelog -also includes a section for changes that have not yet been in a tagged -release. -* @ref md_targets +* @ref CHANGELOG - If this is the web version of the docs, the changelog also + includes a section for changes that have not yet been in a tagged release. +* @ref conventions +* @ref understanding-targets - How all the pieces (`xrt_instance`, IPC, OpenXR) + fit together. * @ref vulkan-extensions -* @ref md_writing-a-new-driver (**not complete**) +* @ref writing-driver (**not complete**) +* @ref ipc-design +* @ref frame-timing +* @ref tracing +* @ref implementing-extension ## Source layout diff --git a/doc/targets.md b/doc/targets.md index 137ede66c..960e47157 100644 --- a/doc/targets.md +++ b/doc/targets.md @@ -1,14 +1,21 @@ -# Understanding and Writing Targets +# Understanding and Writing Targets: Connecting the Pieces {#understanding-targets} -Monado is designed to be a collection of related but independent modules. The -final build product that brings all the desired components together, potentially -with additional code, is called the "target". There are several targets included -in the Monado source tree (in `src/xrt/targets/`) including: +Monado is designed to be a collection of related but independent modules. In a +sense, the Monado project is almost more of a "runtime construction kit" than a +single monolithic runtime. This makes it easy for adaptation and modification, +as well as extension, but it also means that any call in an OpenXR application +goes through quite a few modules before e.g. talking with the driver or the +compositor. + +The final build product that brings all the desired +components together, potentially with additional code, is called the "target". +There are several targets included in the Monado source tree (in +`src/xrt/targets/`) including: - `cli` - builds `monado-cli` executable - `openxr` - builds `libopenxr-monado.so` OpenXR runtime shared object @@ -20,7 +27,8 @@ There is also a directory `common` which builds two static libraries. Because the "target" is responsible for pulling in all the desired drivers, etc. it can lead to some repetition if multiple targets want the same driver collection. For this reason, the "all drivers" code shared between many targets is located here, -though you could consider it a part of the individual targets. +though you could consider it a part of the individual targets. See this section +for details on how the targets find the drivers to probe: @ref writing-driver ## Requirements of a Target @@ -30,7 +38,8 @@ cases, the entry point might be provided by one of the modules being combined to form the target. For instance, an OpenXR runtime must expose `xrNegotiateLoaderRuntimeInterface`: this function is provided by the OpenXR state tracker `st_oxr`, so the OpenXR runtime target just has to link the state -tracker in and ensure it is present in the final build product. +tracker in and ensure that symbol is present and visible in the final build +product. Then, the target must provide access to the collection of devices desired. Target device access is provided by implementing the `xrt_instance` interface in diff --git a/doc/tracing.md b/doc/tracing.md new file mode 100644 index 000000000..8d256bc34 --- /dev/null +++ b/doc/tracing.md @@ -0,0 +1,75 @@ +# Tracing support {#tracing} + + + +## Requirements + +Monado uses the [Perfetto][]/[Percetto][] framework for tracining support, you +need to first build and install [Percetto][] in a place where CMake can find it. +Build [Perfetto][] (you will have gotten the source at least as part of build +[Percetto][]). It is a good idea to familiarise yourself with Perfetto before +proceeding. You then need to build Monado with CMake and give make sure +`XRT_FEATURE_TRACING` is enabled. + +* Build and install [Percetto][]. +* Build and get [Perfetto][] running. +* Build Monado with CMake and with `XRT_FEATURE_TRACING` being `ON`. + +## Running + +Save the following file to `data_events.cfg`, next to your perfetto folder. +Please refer to [Perfetto][] documentation about the format and options of this +config file, but the most important bits is the `tracker_event` section. + +```c +flush_period_ms: 30000 + +incremental_state_config { + clear_period_ms: 500 +} + +buffers: { + size_kb: 63488 + fill_policy: DISCARD +} + +# This is the important bit, this enables all data events from Monado. +data_sources: { + config: { + name: "track_event" + target_buffer: 0 + } +} +``` + +Then run the following commands before launching Monado. + +```bash +# Start the daemon. +# Only needs to be run once and keeps running. +./perfetto/out/linux_clang_release/traced & + +# Start the daemon ftrace probes daemon. +# Only needs to be run once and keeps running. +# Not needed with the above config. +./perfetto/out/linux_clang_release/traced_probes & + +# When you want to run a capture do and then run Monado. +./perfetto/out/linux_clang_release/perfetto --txt -c data_events.cfg -o /tmp/trace.protobuf +``` + +[Perfetto]: https://perfetto.dev +[Percetto]: https://github.com/olvaffe/percetto + +## Gotchas + +Here's where we write down silly things we ran into while using Perfetto/Percetto. + +### "Value doesn't exist" in web viewer + +This is probably because you don't have read permissions on your tracefile, probably because you ran traced/tracebox as root. Don't do that, instead do `sudo chown -R $USER /sys/kernel/tracing` and run traced/tracebox as your normal user. + +(If you really have to run it as root, then before you open the tracefile do `sudo chown $USER `). diff --git a/doc/vulkan-extensions.md b/doc/vulkan-extensions.md index 07d4487a2..7eaa7a8b1 100644 --- a/doc/vulkan-extensions.md +++ b/doc/vulkan-extensions.md @@ -43,6 +43,8 @@ minimize frustration. | [`VK_KHR_external_semaphore`][] (8) (+platform: 7) | yes (soon) ||||||| | [`VK_KHR_swapchain`][] | | yes |||||| +[`VK_EXT_debug_report`][] is also used. + ## Notes Kept out of the table above to limit its width. @@ -85,6 +87,7 @@ Kept out of the table above to limit its width. [`VK_KHR_display`]: https://khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VK_KHR_display.html [`VK_KHR_xcb_surface`]: https://khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VK_KHR_xcb_surface.html [`VK_KHR_wayland_surface`]: https://khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VK_KHR_wayland_surface.html +[`VK_EXT_debug_report`]: https://khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VK_EXT_debug_report.html [`VK_EXT_direct_mode_display`]: https://khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VK_EXT_direct_mode_display.html [`VK_EXT_acquire_xlib_display`]: https://khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VK_EXT_acquire_xlib_display.html [`VK_KHR_android_surface`]: https://khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VK_KHR_android_surface.html diff --git a/doc/writing-a-new-driver.md b/doc/writing-a-new-driver.md index 54893bfac..9ffe69ffe 100644 --- a/doc/writing-a-new-driver.md +++ b/doc/writing-a-new-driver.md @@ -1,4 +1,4 @@ -# Writing a new driver +# Writing a new driver {#writing-driver} + ## About This is a C++ template-based header library for building Kalman-style filters, using [Eigen](http://eigen.tuxfamily.org) linear algebra data types and algorithms. diff --git a/src/external/glad/command.sh b/src/external/glad/command.sh index 7378e0596..8540df6c2 100755 --- a/src/external/glad/command.sh +++ b/src/external/glad/command.sh @@ -35,6 +35,7 @@ EGL_KHR_wait_sync,\ EGL_KHR_image,\ EGL_KHR_image_base,\ EGL_KHR_platform_android,\ +EGL_KHR_no_config_context,\ EGL_EXT_image_gl_colorspace,\ EGL_EXT_image_dma_buf_import,\ EGL_EXT_image_dma_buf_import_modifiers,\ diff --git a/src/external/glad/include/glad/egl.h b/src/external/glad/include/glad/egl.h index 8137c7a28..bbcf3aaa8 100644 --- a/src/external/glad/include/glad/egl.h +++ b/src/external/glad/include/glad/egl.h @@ -1,9 +1,9 @@ /** - * Loader generated by glad 2.0.0-beta on Fri Dec 18 17:45:30 2020 + * Loader generated by glad 2.0.0-beta on Wed Mar 3 17:01:11 2021 * * Generator: C/C++ * Specification: egl - * Extensions: 16 + * Extensions: 17 * * APIs: * - egl=1.4 @@ -18,10 +18,10 @@ * - ON_DEMAND = False * * Commandline: - * --merge --api='egl=1.4' --extensions='EGL_ANDROID_front_buffer_auto_refresh,EGL_ANDROID_get_native_client_buffer,EGL_ANDROID_image_native_buffer,EGL_ANDROID_native_fence_sync,EGL_EXT_image_dma_buf_import,EGL_EXT_image_dma_buf_import_modifiers,EGL_EXT_image_gl_colorspace,EGL_IMG_context_priority,EGL_KHR_create_context,EGL_KHR_fence_sync,EGL_KHR_gl_colorspace,EGL_KHR_image,EGL_KHR_image_base,EGL_KHR_platform_android,EGL_KHR_reusable_sync,EGL_KHR_wait_sync' c + * --merge --api='egl=1.4' --extensions='EGL_ANDROID_front_buffer_auto_refresh,EGL_ANDROID_get_native_client_buffer,EGL_ANDROID_image_native_buffer,EGL_ANDROID_native_fence_sync,EGL_EXT_image_dma_buf_import,EGL_EXT_image_dma_buf_import_modifiers,EGL_EXT_image_gl_colorspace,EGL_IMG_context_priority,EGL_KHR_create_context,EGL_KHR_fence_sync,EGL_KHR_gl_colorspace,EGL_KHR_image,EGL_KHR_image_base,EGL_KHR_no_config_context,EGL_KHR_platform_android,EGL_KHR_reusable_sync,EGL_KHR_wait_sync' c * * Online: - * http://glad.sh/#api=egl%3D1.4&extensions=EGL_ANDROID_front_buffer_auto_refresh%2CEGL_ANDROID_get_native_client_buffer%2CEGL_ANDROID_image_native_buffer%2CEGL_ANDROID_native_fence_sync%2CEGL_EXT_image_dma_buf_import%2CEGL_EXT_image_dma_buf_import_modifiers%2CEGL_EXT_image_gl_colorspace%2CEGL_IMG_context_priority%2CEGL_KHR_create_context%2CEGL_KHR_fence_sync%2CEGL_KHR_gl_colorspace%2CEGL_KHR_image%2CEGL_KHR_image_base%2CEGL_KHR_platform_android%2CEGL_KHR_reusable_sync%2CEGL_KHR_wait_sync&generator=c&options=MERGE + * http://glad.sh/#api=egl%3D1.4&extensions=EGL_ANDROID_front_buffer_auto_refresh%2CEGL_ANDROID_get_native_client_buffer%2CEGL_ANDROID_image_native_buffer%2CEGL_ANDROID_native_fence_sync%2CEGL_EXT_image_dma_buf_import%2CEGL_EXT_image_dma_buf_import_modifiers%2CEGL_EXT_image_gl_colorspace%2CEGL_IMG_context_priority%2CEGL_KHR_create_context%2CEGL_KHR_fence_sync%2CEGL_KHR_gl_colorspace%2CEGL_KHR_image%2CEGL_KHR_image_base%2CEGL_KHR_no_config_context%2CEGL_KHR_platform_android%2CEGL_KHR_reusable_sync%2CEGL_KHR_wait_sync&generator=c&options=MERGE * */ @@ -276,6 +276,7 @@ typedef void (*GLADpostcallback)(void *ret, const char *name, GLADapiproc apipro #define EGL_NONE 0x3038 #define EGL_NON_CONFORMANT_CONFIG 0x3051 #define EGL_NOT_INITIALIZED 0x3001 +#define EGL_NO_CONFIG_KHR EGL_CAST(EGLConfig,0) #define EGL_NO_CONTEXT EGL_CAST(EGLContext,0) #define EGL_NO_DISPLAY EGL_CAST(EGLDisplay,0) #define EGL_NO_IMAGE_KHR EGL_CAST(EGLImageKHR,0) @@ -512,6 +513,8 @@ GLAD_API_CALL int GLAD_EGL_KHR_gl_colorspace; GLAD_API_CALL int GLAD_EGL_KHR_image; #define EGL_KHR_image_base 1 GLAD_API_CALL int GLAD_EGL_KHR_image_base; +#define EGL_KHR_no_config_context 1 +GLAD_API_CALL int GLAD_EGL_KHR_no_config_context; #define EGL_KHR_platform_android 1 GLAD_API_CALL int GLAD_EGL_KHR_platform_android; #define EGL_KHR_reusable_sync 1 diff --git a/src/external/glad/include/glad/gl.h b/src/external/glad/include/glad/gl.h index e24ab563d..8ca8b88c2 100644 --- a/src/external/glad/include/glad/gl.h +++ b/src/external/glad/include/glad/gl.h @@ -1,5 +1,5 @@ /** - * Loader generated by glad 2.0.0-beta on Fri Dec 18 17:45:30 2020 + * Loader generated by glad 2.0.0-beta on Wed Mar 3 17:01:11 2021 * * Generator: C/C++ * Specification: gl diff --git a/src/external/glad/src/egl.c b/src/external/glad/src/egl.c index 33e520659..c21c8259d 100644 --- a/src/external/glad/src/egl.c +++ b/src/external/glad/src/egl.c @@ -38,6 +38,7 @@ int GLAD_EGL_KHR_fence_sync = 0; int GLAD_EGL_KHR_gl_colorspace = 0; int GLAD_EGL_KHR_image = 0; int GLAD_EGL_KHR_image_base = 0; +int GLAD_EGL_KHR_no_config_context = 0; int GLAD_EGL_KHR_platform_android = 0; int GLAD_EGL_KHR_reusable_sync = 0; int GLAD_EGL_KHR_wait_sync = 0; @@ -230,6 +231,7 @@ static int glad_egl_find_extensions_egl(EGLDisplay display) { GLAD_EGL_KHR_gl_colorspace = glad_egl_has_extension(extensions, "EGL_KHR_gl_colorspace"); GLAD_EGL_KHR_image = glad_egl_has_extension(extensions, "EGL_KHR_image"); GLAD_EGL_KHR_image_base = glad_egl_has_extension(extensions, "EGL_KHR_image_base"); + GLAD_EGL_KHR_no_config_context = glad_egl_has_extension(extensions, "EGL_KHR_no_config_context"); GLAD_EGL_KHR_platform_android = glad_egl_has_extension(extensions, "EGL_KHR_platform_android"); GLAD_EGL_KHR_reusable_sync = glad_egl_has_extension(extensions, "EGL_KHR_reusable_sync"); GLAD_EGL_KHR_wait_sync = glad_egl_has_extension(extensions, "EGL_KHR_wait_sync"); diff --git a/src/external/imgui/imgui/cimplot.cpp b/src/external/imgui/imgui/cimplot.cpp index 6228d3425..07d7d19ef 100644 --- a/src/external/imgui/imgui/cimplot.cpp +++ b/src/external/imgui/imgui/cimplot.cpp @@ -1174,9 +1174,8 @@ CIMGUI_API void ImPlot_SetImGuiContext(ImGuiContext* ctx) { return ImPlot::SetImGuiContext(ctx); } -/* + CIMGUI_API void ImPlot_ShowDemoWindow(bool* p_open) { return ImPlot::ShowDemoWindow(p_open); } -*/ \ No newline at end of file diff --git a/src/external/imgui/imgui/implot_demo.cpp b/src/external/imgui/imgui/implot_demo.cpp new file mode 100644 index 000000000..15e139432 --- /dev/null +++ b/src/external/imgui/imgui/implot_demo.cpp @@ -0,0 +1,1552 @@ +// MIT License + +// Copyright (c) 2020 Evan Pezent + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// ImPlot v0.8 WIP + +#include "implot.h" +#include +#include +#include +#include + +#ifdef _MSC_VER +#define sprintf sprintf_s +#endif + +// Encapsulates examples for customizing ImPlot. +namespace MyImPlot { + +// Example for Custom Data and Getters section. +struct Vector2f { + Vector2f(float _x, float _y) { x = _x; y = _y; } + float x, y; +}; + +// Example for Custom Data and Getters section. +struct WaveData { + double X, Amp, Freq, Offset; + WaveData(double x, double amp, double freq, double offset) { X = x; Amp = amp; Freq = freq; Offset = offset; } +}; +ImPlotPoint SineWave(void* wave_data, int idx); +ImPlotPoint SawWave(void* wave_data, int idx); +ImPlotPoint Spiral(void*, int idx); + +// Example for Tables section. +void Sparkline(const char* id, const float* values, int count, float min_v, float max_v, int offset, const ImVec4& col, const ImVec2& size); + +// Example for Custom Plotters and Tooltips section. +void PlotCandlestick(const char* label_id, const double* xs, const double* opens, const double* closes, const double* lows, const double* highs, int count, bool tooltip = true, float width_percent = 0.25f, ImVec4 bullCol = ImVec4(0,1,0,1), ImVec4 bearCol = ImVec4(1,0,0,1)); + +// Example for Custom Styles section. +void StyleSeaborn(); + +} // namespace MyImPlot + +namespace ImPlot { + +void ShowBenchmarkTool(); + +template +inline T RandomRange(T min, T max) { + T scale = rand() / (T) RAND_MAX; + return min + scale * ( max - min ); +} + +// utility structure for realtime plot +struct ScrollingBuffer { + int MaxSize; + int Offset; + ImVector Data; + ScrollingBuffer() { + MaxSize = 2000; + Offset = 0; + Data.reserve(MaxSize); + } + void AddPoint(float x, float y) { + if (Data.size() < MaxSize) + Data.push_back(ImVec2(x,y)); + else { + Data[Offset] = ImVec2(x,y); + Offset = (Offset + 1) % MaxSize; + } + } + void Erase() { + if (Data.size() > 0) { + Data.shrink(0); + Offset = 0; + } + } +}; + +// utility structure for realtime plot +struct RollingBuffer { + float Span; + ImVector Data; + RollingBuffer() { + Span = 10.0f; + Data.reserve(2000); + } + void AddPoint(float x, float y) { + float xmod = fmodf(x, Span); + if (!Data.empty() && xmod < Data.back().x) + Data.shrink(0); + Data.push_back(ImVec2(xmod, y)); + } +}; + +// Huge data used by Time Formatting example (~500 MB allocation!) +struct HugeTimeData { + HugeTimeData(double min) { + Ts = new double[Size]; + Ys = new double[Size]; + for (int i = 0; i < Size; ++i) { + Ts[i] = min + i; + Ys[i] = GetY(Ts[i]); + } + } + ~HugeTimeData() { delete[] Ts; delete[] Ys; } + static double GetY(double t) { + return 0.5 + 0.25 * sin(t/86400/12) + 0.005 * sin(t/3600); + } + double* Ts; + double* Ys; + static const int Size = 60*60*24*366; +}; + +void ShowDemoWindow(bool* p_open) { + double DEMO_TIME = ImGui::GetTime(); + static bool show_imgui_metrics = false; + static bool show_imgui_style_editor = false; + static bool show_implot_style_editor = false; + static bool show_implot_benchmark = false; + if (show_imgui_metrics) { + ImGui::ShowMetricsWindow(&show_imgui_metrics); + } + if (show_imgui_style_editor) { + ImGui::Begin("Style Editor (ImGui)", &show_imgui_style_editor); + ImGui::ShowStyleEditor(); + ImGui::End(); + } + if (show_implot_style_editor) { + ImGui::SetNextWindowSize(ImVec2(415,762), ImGuiCond_Appearing); + ImGui::Begin("Style Editor (ImPlot)", &show_implot_style_editor); + ImPlot::ShowStyleEditor(); + ImGui::End(); + } + if (show_implot_benchmark) { + ImGui::SetNextWindowSize(ImVec2(530,740), ImGuiCond_Appearing); + ImGui::Begin("ImPlot Benchmark Tool", &show_implot_benchmark); + ImPlot::ShowBenchmarkTool(); + ImGui::End(); + return; + } + ImGui::SetNextWindowPos(ImVec2(50, 50), ImGuiCond_FirstUseEver); + ImGui::SetNextWindowSize(ImVec2(600, 750), ImGuiCond_FirstUseEver); + ImGui::Begin("ImPlot Demo", p_open, ImGuiWindowFlags_MenuBar); + if (ImGui::BeginMenuBar()) { + if (ImGui::BeginMenu("Tools")) { + ImGui::MenuItem("Metrics (ImGui)", NULL, &show_imgui_metrics); + ImGui::MenuItem("Style Editor (ImGui)", NULL, &show_imgui_style_editor); + ImGui::MenuItem("Style Editor (ImPlot)", NULL, &show_implot_style_editor); + ImGui::MenuItem("Benchmark", NULL, &show_implot_benchmark); + ImGui::EndMenu(); + } + ImGui::EndMenuBar(); + } + //------------------------------------------------------------------------- + ImGui::Text("ImPlot says hello. (%s)", IMPLOT_VERSION); + ImGui::Spacing(); + + if (ImGui::CollapsingHeader("Help")) { + ImGui::Text("ABOUT THIS DEMO:"); + ImGui::BulletText("Sections below are demonstrating many aspects of the library."); + ImGui::BulletText("The \"Tools\" menu above gives access to: Style Editors (ImPlot/ImGui)\n" + "and Metrics (general purpose Dear ImGui debugging tool)."); + ImGui::Separator(); + ImGui::Text("PROGRAMMER GUIDE:"); + ImGui::BulletText("See the ShowDemoWindow() code in implot_demo.cpp. <- you are here!"); + ImGui::BulletText("By default, anti-aliased lines are turned OFF."); + ImGui::Indent(); + ImGui::BulletText("Software AA can be enabled globally with ImPlotStyle.AntiAliasedLines."); + ImGui::BulletText("Software AA can be enabled per plot with ImPlotFlags_AntiAliased."); + ImGui::BulletText("AA for plots can be toggled from the plot's context menu."); + ImGui::BulletText("If permitable, you are better off using hardware AA (e.g. MSAA)."); + ImGui::Unindent(); + ImGui::BulletText("If you see visual artifacts, do one of the following:"); + ImGui::Indent(); + ImGui::BulletText("Handle ImGuiBackendFlags_RendererHasVtxOffset for 16-bit indices in your backend."); + ImGui::BulletText("Or, enable 32-bit indices in imconfig.h."); + ImGui::BulletText("Your current configuration is:"); + ImGui::Indent(); + ImGui::BulletText("ImDrawIdx: %d-bit", (int)(sizeof(ImDrawIdx) * 8)); + ImGui::BulletText("ImGuiBackendFlags_RendererHasVtxOffset: %s", (ImGui::GetIO().BackendFlags & ImGuiBackendFlags_RendererHasVtxOffset) ? "True" : "False"); + ImGui::Unindent(); + ImGui::Unindent(); +#ifdef IMPLOT_DEMO_USE_DOUBLE + ImGui::BulletText("The demo data precision is: double"); +#else + ImGui::BulletText("The demo data precision is: float"); +#endif + ImGui::Separator(); + ImGui::Text("USER GUIDE:"); + ShowUserGuide(); + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Configuration")) { + ImGui::ShowFontSelector("Font"); + ImGui::ShowStyleSelector("ImGui Style"); + ImPlot::ShowStyleSelector("ImPlot Style"); + + static const char* map = ImPlot::GetColormapName(ImPlotColormap_Default); + if (ImGui::BeginCombo("ImPlot Colormap", map)) { + for (int i = 0; i < ImPlotColormap_COUNT; ++i) { + const char* name = GetColormapName(i); + if (ImGui::Selectable(name, map == name)) { + map = name; + ImPlot::SetColormap(i); + } + } + ImGui::EndCombo(); + } + float indent = ImGui::CalcItemWidth() - ImGui::GetFrameHeight(); + ImGui::Indent(ImGui::CalcItemWidth() - ImGui::GetFrameHeight()); + ImGui::Checkbox("Anti-Aliased Lines", &ImPlot::GetStyle().AntiAliasedLines); + ImGui::Unindent(indent); + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Line Plots")) { + static float xs1[1001], ys1[1001]; + for (int i = 0; i < 1001; ++i) { + xs1[i] = i * 0.001f; + ys1[i] = 0.5f + 0.5f * sinf(50 * (xs1[i] + (float)DEMO_TIME / 10)); + } + static double xs2[11], ys2[11]; + for (int i = 0; i < 11; ++i) { + xs2[i] = i * 0.1f; + ys2[i] = xs2[i] * xs2[i]; + } + ImGui::BulletText("Anti-aliasing can be enabled from the plot's context menu (see Help)."); + if (ImPlot::BeginPlot("Line Plot", "x", "f(x)")) { + ImPlot::PlotLine("sin(x)", xs1, ys1, 1001); + ImPlot::SetNextMarkerStyle(ImPlotMarker_Circle); + ImPlot::PlotLine("x^2", xs2, ys2, 11); + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Filled Line Plots")) { + static double xs1[101], ys1[101], ys2[101], ys3[101]; + srand(0); + for (int i = 0; i < 101; ++i) { + xs1[i] = (float)i; + ys1[i] = RandomRange(400.0,450.0); + ys2[i] = RandomRange(275.0,350.0); + ys3[i] = RandomRange(150.0,225.0); + } + static bool show_lines = true; + static bool show_fills = true; + static float fill_ref = 0; + ImGui::Checkbox("Lines",&show_lines); ImGui::SameLine(); + ImGui::Checkbox("Fills",&show_fills); + ImGui::DragFloat("Reference",&fill_ref, 1, -100, 500); + + ImPlot::SetNextPlotLimits(0,100,0,500); + if (ImPlot::BeginPlot("Stock Prices", "Days", "Price")) { + if (show_fills) { + ImPlot::PushStyleVar(ImPlotStyleVar_FillAlpha, 0.25f); + ImPlot::PlotShaded("Stock 1", xs1, ys1, 101, fill_ref); + ImPlot::PlotShaded("Stock 2", xs1, ys2, 101, fill_ref); + ImPlot::PlotShaded("Stock 3", xs1, ys3, 101, fill_ref); + ImPlot::PopStyleVar(); + } + if (show_lines) { + ImPlot::PlotLine("Stock 1", xs1, ys1, 101); + ImPlot::PlotLine("Stock 2", xs1, ys2, 101); + ImPlot::PlotLine("Stock 3", xs1, ys3, 101); + } + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Shaded Plots")) { + static float xs[1001], ys[1001], ys1[1001], ys2[1001], ys3[1001], ys4[1001]; + srand(0); + for (int i = 0; i < 1001; ++i) { + xs[i] = i * 0.001f; + ys[i] = 0.25f + 0.25f * sinf(25 * xs[i]) * sinf(5 * xs[i]) + RandomRange(-0.01f, 0.01f); + ys1[i] = ys[i] + RandomRange(0.1f, 0.12f); + ys2[i] = ys[i] - RandomRange(0.1f, 0.12f); + ys3[i] = 0.75f + 0.2f * sinf(25 * xs[i]); + ys4[i] = 0.75f + 0.1f * cosf(25 * xs[i]); + } + static float alpha = 0.25f; + ImGui::DragFloat("Alpha",&alpha,0.01f,0,1); + + if (ImPlot::BeginPlot("Shaded Plots", "X-Axis", "Y-Axis")) { + ImPlot::PushStyleVar(ImPlotStyleVar_FillAlpha, alpha); + ImPlot::PlotShaded("Uncertain Data",xs,ys1,ys2,1001); + ImPlot::PlotLine("Uncertain Data", xs, ys, 1001); + ImPlot::PlotShaded("Overlapping",xs,ys3,ys4,1001); + ImPlot::PlotLine("Overlapping",xs,ys3,1001); + ImPlot::PlotLine("Overlapping",xs,ys4,1001); + ImPlot::PopStyleVar(); + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Scatter Plots")) { + srand(0); + static float xs1[100], ys1[100]; + for (int i = 0; i < 100; ++i) { + xs1[i] = i * 0.01f; + ys1[i] = xs1[i] + 0.1f * ((float)rand() / (float)RAND_MAX); + } + static float xs2[50], ys2[50]; + for (int i = 0; i < 50; i++) { + xs2[i] = 0.25f + 0.2f * ((float)rand() / (float)RAND_MAX); + ys2[i] = 0.75f + 0.2f * ((float)rand() / (float)RAND_MAX); + } + + if (ImPlot::BeginPlot("Scatter Plot", NULL, NULL)) { + ImPlot::PlotScatter("Data 1", xs1, ys1, 100); + ImPlot::PushStyleVar(ImPlotStyleVar_FillAlpha, 0.25f); + ImPlot::SetNextMarkerStyle(ImPlotMarker_Square, 6, ImVec4(0,1,0,0.5f), IMPLOT_AUTO, ImVec4(0,1,0,1)); + ImPlot::PlotScatter("Data 2", xs2, ys2, 50); + ImPlot::PopStyleVar(); + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Bar Plots")) { + + static bool horz = false; + static ImS8 midtm[10] = {83, 67, 23, 89, 83, 78, 91, 82, 85, 90}; + static ImS16 final[10] = {80, 62, 56, 99, 55, 78, 88, 78, 90, 100}; + static ImS32 grade[10] = {80, 69, 52, 92, 72, 78, 75, 76, 89, 95}; + + static const char* labels[] = {"S1","S2","S3","S4","S5","S6","S7","S8","S9","S10"}; + static const double positions[] = {0,1,2,3,4,5,6,7,8,9}; + + ImGui::Checkbox("Horizontal",&horz); + + if (horz) { + ImPlot::SetNextPlotLimits(0, 110, -0.5, 9.5, ImGuiCond_Always); + ImPlot::SetNextPlotTicksY(positions, 10, labels); + } + else { + ImPlot::SetNextPlotLimits(-0.5, 9.5, 0, 110, ImGuiCond_Always); + ImPlot::SetNextPlotTicksX(positions, 10, labels); + } + if (ImPlot::BeginPlot("Bar Plot", horz ? "Score" : "Student", horz ? "Student" : "Score", + ImVec2(-1,0), 0, 0, horz ? ImPlotAxisFlags_Invert : 0)) + { + if (horz) { + ImPlot::PlotBarsH("Midterm Exam", midtm, 10, 0.2, -0.2); + ImPlot::PlotBarsH("Final Exam", final, 10, 0.2, 0); + ImPlot::PlotBarsH("Course Grade", grade, 10, 0.2, 0.2); + } + else { + ImPlot::PlotBars("Midterm Exam", midtm, 10, 0.2, -0.2); + ImPlot::PlotBars("Final Exam", final, 10, 0.2, 0); + ImPlot::PlotBars("Course Grade", grade, 10, 0.2, 0.2); + } + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Error Bars")) { + static float xs[5] = {1,2,3,4,5}; + static float bar[5] = {1,2,5,3,4}; + static float lin1[5] = {8,8,9,7,8}; + static float lin2[5] = {6,7,6,9,6}; + static float err1[5] = {0.2f, 0.4f, 0.2f, 0.6f, 0.4f}; + static float err2[5] = {0.4f, 0.2f, 0.4f, 0.8f, 0.6f}; + static float err3[5] = {0.09f, 0.14f, 0.09f, 0.12f, 0.16f}; + static float err4[5] = {0.02f, 0.08f, 0.15f, 0.05f, 0.2f}; + + + ImPlot::SetNextPlotLimits(0, 6, 0, 10); + if (ImPlot::BeginPlot("##ErrorBars",NULL,NULL)) { + + ImPlot::PlotBars("Bar", xs, bar, 5, 0.5f); + ImPlot::PlotErrorBars("Bar", xs, bar, err1, 5); + + ImPlot::SetNextErrorBarStyle(ImPlot::GetColormapColor(1), 0); + ImPlot::PlotErrorBars("Line", xs, lin1, err1, err2, 5); + ImPlot::SetNextMarkerStyle(ImPlotMarker_Circle); + ImPlot::PlotLine("Line", xs, lin1, 5); + + ImPlot::PushStyleColor(ImPlotCol_ErrorBar, ImPlot::GetColormapColor(2)); + ImPlot::PlotErrorBars("Scatter", xs, lin2, err2, 5); + ImPlot::PlotErrorBarsH("Scatter", xs, lin2, err3, err4, 5); + ImPlot::PopStyleColor(); + ImPlot::PlotScatter("Scatter", xs, lin2, 5); + + ImPlot::EndPlot(); + } + } + if (ImGui::CollapsingHeader("Stem Plots")) { + static double xs[51], ys1[51], ys2[51]; + for (int i = 0; i < 51; ++i) { + xs[i] = i * 0.02; + ys1[i] = 1.0 + 0.5 * sin(25*xs[i])*cos(2*xs[i]); + ys2[i] = 0.5 + 0.25 * sin(10*xs[i]) * sin(xs[i]); + } + ImPlot::SetNextPlotLimits(0,1,0,1.6); + if (ImPlot::BeginPlot("Stem Plots")) { + + ImPlot::PlotStems("Stems 1",xs,ys1,51); + + ImPlot::SetNextLineStyle(ImVec4(1,0.5f,0,0.75f)); + ImPlot::SetNextMarkerStyle(ImPlotMarker_Square,5,ImVec4(1,0.5f,0,0.25f)); + ImPlot::PlotStems("Stems 2", xs, ys2,51); + + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Pie Charts")) { + static const char* labels1[] = {"Frogs","Hogs","Dogs","Logs"}; + static float data1[] = {0.15f, 0.30f, 0.2f, 0.05f}; + static bool normalize = false; + ImGui::SetNextItemWidth(250); + ImGui::DragFloat4("Values", data1, 0.01f, 0, 1); + if ((data1[0] + data1[1] + data1[2] + data1[3]) < 1) { + ImGui::SameLine(); + ImGui::Checkbox("Normalize", &normalize); + } + + ImPlot::SetNextPlotLimits(0,1,0,1,ImGuiCond_Always); + if (ImPlot::BeginPlot("##Pie1", NULL, NULL, ImVec2(250,250), ImPlotFlags_NoMousePos, ImPlotAxisFlags_NoDecorations, ImPlotAxisFlags_NoDecorations)) { + ImPlot::PlotPieChart(labels1, data1, 4, 0.5, 0.5, 0.4, normalize, "%.2f"); + ImPlot::EndPlot(); + } + + ImGui::SameLine(); + + static const char* labels2[] = {"A","B","C","D","E"}; + static int data2[] = {1,1,2,3,5}; + + ImPlot::PushColormap(ImPlotColormap_Pastel); + ImPlot::SetNextPlotLimits(0,1,0,1,ImGuiCond_Always); + if (ImPlot::BeginPlot("##Pie2", NULL, NULL, ImVec2(250,250), ImPlotFlags_NoMousePos, ImPlotAxisFlags_NoDecorations, ImPlotAxisFlags_NoDecorations)) { + ImPlot::PlotPieChart(labels2, data2, 5, 0.5, 0.5, 0.4, true, "%.0f", 180); + ImPlot::EndPlot(); + } + ImPlot::PopColormap(); + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Heatmaps")) { + static float values1[7][7] = {{0.8f, 2.4f, 2.5f, 3.9f, 0.0f, 4.0f, 0.0f}, + {2.4f, 0.0f, 4.0f, 1.0f, 2.7f, 0.0f, 0.0f}, + {1.1f, 2.4f, 0.8f, 4.3f, 1.9f, 4.4f, 0.0f}, + {0.6f, 0.0f, 0.3f, 0.0f, 3.1f, 0.0f, 0.0f}, + {0.7f, 1.7f, 0.6f, 2.6f, 2.2f, 6.2f, 0.0f}, + {1.3f, 1.2f, 0.0f, 0.0f, 0.0f, 3.2f, 5.1f}, + {0.1f, 2.0f, 0.0f, 1.4f, 0.0f, 1.9f, 6.3f}}; + static float scale_min = 0; + static float scale_max = 6.3f; + static const char* xlabels[] = {"C1","C2","C3","C4","C5","C6","C7"}; + static const char* ylabels[] = {"R1","R2","R3","R4","R5","R6","R7"}; + + static ImPlotColormap map = ImPlotColormap_Viridis; + if (ImGui::Button("Change Colormap",ImVec2(225,0))) + map = (map + 1) % ImPlotColormap_COUNT; + + ImGui::SameLine(); + ImGui::LabelText("##Colormap Index", "%s", ImPlot::GetColormapName(map)); + ImGui::SetNextItemWidth(225); + ImGui::DragFloatRange2("Min / Max",&scale_min, &scale_max, 0.01f, -20, 20); + static ImPlotAxisFlags axes_flags = ImPlotAxisFlags_Lock | ImPlotAxisFlags_NoGridLines | ImPlotAxisFlags_NoTickMarks; + + ImPlot::PushColormap(map); + SetNextPlotTicksX(0 + 1.0/14.0, 1 - 1.0/14.0, 7, xlabels); + SetNextPlotTicksY(1 - 1.0/14.0, 0 + 1.0/14.0, 7, ylabels); + if (ImPlot::BeginPlot("##Heatmap1",NULL,NULL,ImVec2(225,225),ImPlotFlags_NoLegend|ImPlotFlags_NoMousePos,axes_flags,axes_flags)) { + ImPlot::PlotHeatmap("heat",values1[0],7,7,scale_min,scale_max); + ImPlot::EndPlot(); + } + ImGui::SameLine(); + ImPlot::ShowColormapScale(scale_min, scale_max, 225); + ImPlot::PopColormap(); + + ImGui::SameLine(); + + static double values2[100*100]; + srand((unsigned int)(DEMO_TIME*1000000)); + for (int i = 0; i < 100*100; ++i) + values2[i] = RandomRange(0.0,1.0); + + static ImVec4 gray[2] = {ImVec4(0,0,0,1), ImVec4(1,1,1,1)}; + ImPlot::PushColormap(gray, 2); + ImPlot::SetNextPlotLimits(-1,1,-1,1); + if (ImPlot::BeginPlot("##Heatmap2",NULL,NULL,ImVec2(225,225),0,ImPlotAxisFlags_NoDecorations,ImPlotAxisFlags_NoDecorations)) { + ImPlot::PlotHeatmap("heat1",values2,100,100,0,1,NULL); + ImPlot::PlotHeatmap("heat2",values2,100,100,0,1,NULL, ImPlotPoint(-1,-1), ImPlotPoint(0,0)); + ImPlot::EndPlot(); + } + ImPlot::PopColormap(); + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Realtime Plots")) { + ImGui::BulletText("Move your mouse to change the data!"); + ImGui::BulletText("This example assumes 60 FPS. Higher FPS requires larger buffer size."); + static ScrollingBuffer sdata1, sdata2; + static RollingBuffer rdata1, rdata2; + ImVec2 mouse = ImGui::GetMousePos(); + static float t = 0; + t += ImGui::GetIO().DeltaTime; + sdata1.AddPoint(t, mouse.x * 0.0005f); + rdata1.AddPoint(t, mouse.x * 0.0005f); + sdata2.AddPoint(t, mouse.y * 0.0005f); + rdata2.AddPoint(t, mouse.y * 0.0005f); + + static float history = 10.0f; + ImGui::SliderFloat("History",&history,1,30,"%.1f s"); + rdata1.Span = history; + rdata2.Span = history; + + static ImPlotAxisFlags rt_axis = ImPlotAxisFlags_NoTickLabels; + ImPlot::SetNextPlotLimitsX(t - history, t, ImGuiCond_Always); + if (ImPlot::BeginPlot("##Scrolling", NULL, NULL, ImVec2(-1,150), 0, rt_axis, rt_axis | ImPlotAxisFlags_LockMin)) { + ImPlot::PlotShaded("Data 1", &sdata1.Data[0].x, &sdata1.Data[0].y, sdata1.Data.size(), 0, sdata1.Offset, 2 * sizeof(float)); + ImPlot::PlotLine("Data 2", &sdata2.Data[0], sdata2.Data.size(), sdata2.Offset); + ImPlot::EndPlot(); + } + ImPlot::SetNextPlotLimitsX(0, history, ImGuiCond_Always); + if (ImPlot::BeginPlot("##Rolling", NULL, NULL, ImVec2(-1,150), 0, rt_axis, rt_axis)) { + // two methods of plotting Data + // as ImVec2* (or ImPlot*): + ImPlot::PlotLine("Data 1", &rdata1.Data[0], rdata1.Data.size()); + // as float*, float* (or double*, double*) with stride of 2 * sizeof + ImPlot::PlotLine("Data 2", &rdata2.Data[0].x, &rdata2.Data[0].y, rdata2.Data.size(), 0, 2 * sizeof(float)); + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Markers and Text")) { + static float mk_size = ImPlot::GetStyle().MarkerSize; + static float mk_weight = ImPlot::GetStyle().MarkerWeight; + ImGui::DragFloat("Marker Size",&mk_size,0.1f,2.0f,10.0f,"%.2f px"); + ImGui::DragFloat("Marker Weight", &mk_weight,0.05f,0.5f,3.0f,"%.2f px"); + + ImPlot::SetNextPlotLimits(0, 10, 0, 12); + if (ImPlot::BeginPlot("##MarkerStyles", NULL, NULL, ImVec2(-1,0), ImPlotFlags_CanvasOnly, ImPlotAxisFlags_NoDecorations, ImPlotAxisFlags_NoDecorations)) { + ImS8 xs[2] = {1,4}; + ImS8 ys[2] = {10,11}; + + // filled markers + for (int m = 1; m < ImPlotMarker_COUNT; ++m) { + ImGui::PushID(m); + ImPlot::SetNextMarkerStyle(m, mk_size, IMPLOT_AUTO_COL, mk_weight); + ImPlot::PlotLine("##Filled", xs, ys, 2); + ImGui::PopID(); + ys[0]--; ys[1]--; + } + xs[0] = 6; xs[1] = 9; ys[0] = 10; ys[1] = 11; + // open markers + for (int m = 1; m < ImPlotMarker_COUNT; ++m) { + ImGui::PushID(m); + ImPlot::SetNextMarkerStyle(m, mk_size, ImVec4(0,0,0,0), mk_weight); + ImPlot::PlotLine("##Open", xs, ys, 2); + ImGui::PopID(); + ys[0]--; ys[1]--; + } + + ImPlot::PlotText("Filled Markers", 2.5f, 6.0f); + ImPlot::PlotText("Open Markers", 7.5f, 6.0f); + + ImPlot::PushStyleColor(ImPlotCol_InlayText, ImVec4(1,0,1,1)); + ImPlot::PlotText("Vertical Text", 5.0f, 6.0f, true); + ImPlot::PopStyleColor(); + + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Log Scale")) { + static double xs[1001], ys1[1001], ys2[1001], ys3[1001]; + for (int i = 0; i < 1001; ++i) { + xs[i] = i*0.1f; + ys1[i] = sin(xs[i]) + 1; + ys2[i] = log(xs[i]); + ys3[i] = pow(10.0, xs[i]); + } + ImGui::BulletText("Open the plot context menu (double right click) to change scales."); + + ImPlot::SetNextPlotLimits(0.1, 100, 0, 10); + if (ImPlot::BeginPlot("Log Plot", NULL, NULL, ImVec2(-1,0), 0, ImPlotAxisFlags_LogScale )) { + ImPlot::PlotLine("f(x) = x", xs, xs, 1001); + ImPlot::PlotLine("f(x) = sin(x)+1", xs, ys1, 1001); + ImPlot::PlotLine("f(x) = log(x)", xs, ys2, 1001); + ImPlot::PlotLine("f(x) = 10^x", xs, ys3, 21); + ImPlot::EndPlot(); + } + } + if (ImGui::CollapsingHeader("Time Formatted Axes")) { + + static double t_min = 1577836800; // 01/01/2020 @ 12:00:00am (UTC) + static double t_max = 1609459200; // 01/01/2021 @ 12:00:00am (UTC) + + ImGui::BulletText("When ImPlotAxisFlags_Time is enabled on the X-Axis, values are interpreted as\n" + "UNIX timestamps in seconds and axis labels are formated as date/time."); + ImGui::BulletText("By default, labels are in UTC time but can be set to use local time instead."); + + ImGui::Checkbox("Use Local Time",&ImPlot::GetStyle().UseLocalTime); + + static HugeTimeData* data = NULL; + if (data == NULL) { + ImGui::SameLine(); + if (ImGui::Button("Generate Huge Data (~500MB!)")) { + static HugeTimeData sdata(t_min); + data = &sdata; + } + } + + ImPlot::SetNextPlotLimits(t_min,t_max,0,1); + if (ImPlot::BeginPlot("##Time", NULL, NULL, ImVec2(-1,0), 0, ImPlotAxisFlags_Time)) { + if (data != NULL) { + // downsample our data + int downsample = (int)ImPlot::GetPlotLimits().X.Size() / 1000 + 1; + int start = (int)(ImPlot::GetPlotLimits().X.Min - t_min); + start = start < 0 ? 0 : start > HugeTimeData::Size - 1 ? HugeTimeData::Size - 1 : start; + int end = (int)(ImPlot::GetPlotLimits().X.Max - t_min) + 1000; + end = end < 0 ? 0 : end > HugeTimeData::Size - 1 ? HugeTimeData::Size - 1 : end; + int size = (end - start)/downsample; + // plot it + ImPlot::PlotLine("Time Series", &data->Ts[start], &data->Ys[start], size, 0, sizeof(double)*downsample); + } + // plot time now + double t_now = (double)time(0); + double y_now = HugeTimeData::GetY(t_now); + ImPlot::PlotScatter("Now",&t_now,&y_now,1); + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Multiple Y-Axes")) { + static float xs[1001], xs2[1001], ys1[1001], ys2[1001], ys3[1001]; + for (int i = 0; i < 1001; ++i) { + xs[i] = (i*0.1f); + ys1[i] = sinf(xs[i]) * 3 + 1; + ys2[i] = cosf(xs[i]) * 0.2f + 0.5f; + ys3[i] = sinf(xs[i]+0.5f) * 100 + 200; + xs2[i] = xs[i] + 10.0f; + } + static bool y2_axis = true; + static bool y3_axis = true; + ImGui::Checkbox("Y-Axis 2", &y2_axis); + ImGui::SameLine(); + ImGui::Checkbox("Y-Axis 3", &y3_axis); + ImGui::SameLine(); + + // you can fit axes programatically + ImGui::SameLine(); if (ImGui::Button("Fit X")) ImPlot::FitNextPlotAxes(true, false, false, false); + ImGui::SameLine(); if (ImGui::Button("Fit Y")) ImPlot::FitNextPlotAxes(false, true, false, false); + ImGui::SameLine(); if (ImGui::Button("Fit Y2")) ImPlot::FitNextPlotAxes(false, false, true, false); + ImGui::SameLine(); if (ImGui::Button("Fit Y3")) ImPlot::FitNextPlotAxes(false, false, false, true); + + ImPlot::SetNextPlotLimits(0.1, 100, 0, 10); + ImPlot::SetNextPlotLimitsY(0, 1, ImGuiCond_Once, 1); + ImPlot::SetNextPlotLimitsY(0, 300, ImGuiCond_Once, 2); + if (ImPlot::BeginPlot("Multi-Axis Plot", NULL, NULL, ImVec2(-1,0), + (y2_axis ? ImPlotFlags_YAxis2 : 0) | + (y3_axis ? ImPlotFlags_YAxis3 : 0))) { + ImPlot::PlotLine("f(x) = x", xs, xs, 1001); + ImPlot::PlotLine("f(x) = sin(x)*3+1", xs, ys1, 1001); + if (y2_axis) { + ImPlot::SetPlotYAxis(1); + ImPlot::PlotLine("f(x) = cos(x)*.2+.5 (Y2)", xs, ys2, 1001); + } + if (y3_axis) { + ImPlot::SetPlotYAxis(2); + ImPlot::PlotLine("f(x) = sin(x+.5)*100+200 (Y3)", xs2, ys3, 1001); + } + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Linked Axes")) { + static double xmin = 0, xmax = 1, ymin = 0, ymax = 1; + static bool linkx = true, linky = true; + int data[2] = {0,1}; + ImGui::Checkbox("Link X", &linkx); + ImGui::SameLine(); + ImGui::Checkbox("Link Y", &linky); + ImPlot::LinkNextPlotLimits(linkx ? &xmin : NULL , linkx ? &xmax : NULL, linky ? &ymin : NULL, linky ? &ymax : NULL); + if (ImPlot::BeginPlot("Plot A")) { + ImPlot::PlotLine("Line",data,2); + ImPlot::EndPlot(); + } + ImPlot::LinkNextPlotLimits(linkx ? &xmin : NULL , linkx ? &xmax : NULL, linky ? &ymin : NULL, linky ? &ymax : NULL); + if (ImPlot::BeginPlot("Plot B")) { + ImPlot::PlotLine("Line",data,2); + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Querying")) { + static ImVector data; + static ImPlotLimits range, query; + + ImGui::BulletText("Ctrl + click in the plot area to draw points."); + ImGui::BulletText("Middle click (or Ctrl + right click) and drag to create a query rect."); + ImGui::Indent(); + ImGui::BulletText("Hold Alt to expand query horizontally."); + ImGui::BulletText("Hold Shift to expand query vertically."); + ImGui::BulletText("The query rect can be dragged after it's created."); + ImGui::Unindent(); + + if (ImPlot::BeginPlot("##Drawing", NULL, NULL, ImVec2(-1,0), ImPlotFlags_Query, ImPlotAxisFlags_NoDecorations, ImPlotAxisFlags_NoDecorations)) { + if (ImPlot::IsPlotHovered() && ImGui::IsMouseClicked(0) && ImGui::GetIO().KeyCtrl) { + ImPlotPoint pt = ImPlot::GetPlotMousePos(); + data.push_back(pt); + } + if (data.size() > 0) + ImPlot::PlotScatter("Points", &data[0].x, &data[0].y, data.size(), 0, 2 * sizeof(double)); + if (ImPlot::IsPlotQueried() && data.size() > 0) { + ImPlotLimits range2 = ImPlot::GetPlotQuery(); + int cnt = 0; + ImPlotPoint avg; + for (int i = 0; i < data.size(); ++i) { + if (range2.Contains(data[i].x, data[i].y)) { + avg.x += data[i].x; + avg.y += data[i].y; + cnt++; + } + } + if (cnt > 0) { + avg.x = avg.x / cnt; + avg.y = avg.y / cnt; + ImPlot::SetNextMarkerStyle(ImPlotMarker_Square); + ImPlot::PlotScatter("Average", &avg.x, &avg.y, 1); + } + } + range = ImPlot::GetPlotLimits(); + query = ImPlot::GetPlotQuery(); + ImPlot::EndPlot(); + } + ImGui::Text("The current plot limits are: [%g,%g,%g,%g]", range.X.Min, range.X.Max, range.Y.Min, range.Y.Max); + ImGui::Text("The current query limits are: [%g,%g,%g,%g]", query.X.Min, query.X.Max, query.Y.Min, query.Y.Max); + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Views")) { + // mimic's soulthread's imgui_plot demo + static float x_data[512]; + static float y_data1[512]; + static float y_data2[512]; + static float y_data3[512]; + static float sampling_freq = 44100; + static float freq = 500; + for (size_t i = 0; i < 512; ++i) { + const float t = i / sampling_freq; + x_data[i] = t; + const float arg = 2 * 3.14f * freq * t; + y_data1[i] = sinf(arg); + y_data2[i] = y_data1[i] * -0.6f + sinf(2 * arg) * 0.4f; + y_data3[i] = y_data2[i] * -0.6f + sinf(3 * arg) * 0.4f; + } + ImGui::BulletText("Query the first plot to render a subview in the second plot (see above for controls)."); + ImPlot::SetNextPlotLimits(0,0.01,-1,1); + ImPlotAxisFlags flags = ImPlotAxisFlags_NoTickLabels; + ImPlotLimits query; + if (ImPlot::BeginPlot("##View1",NULL,NULL,ImVec2(-1,150), ImPlotFlags_Query, flags, flags)) { + ImPlot::PlotLine("Signal 1", x_data, y_data1, 512); + ImPlot::PlotLine("Signal 2", x_data, y_data2, 512); + ImPlot::PlotLine("Signal 3", x_data, y_data3, 512); + query = ImPlot::GetPlotQuery(); + ImPlot::EndPlot(); + } + ImPlot::SetNextPlotLimits(query.X.Min, query.X.Max, query.Y.Min, query.Y.Max, ImGuiCond_Always); + if (ImPlot::BeginPlot("##View2",NULL,NULL,ImVec2(-1,150), ImPlotFlags_CanvasOnly, ImPlotAxisFlags_NoDecorations, ImPlotAxisFlags_NoDecorations)) { + ImPlot::PlotLine("Signal 1", x_data, y_data1, 512); + ImPlot::PlotLine("Signal 2", x_data, y_data2, 512); + ImPlot::PlotLine("Signal 3", x_data, y_data3, 512); + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Drag and Drop")) { + const int K_CHANNELS = 9; + srand((int)(10000000 * DEMO_TIME)); + static bool paused = false; + static bool init = true; + static ScrollingBuffer data[K_CHANNELS]; + static bool show[K_CHANNELS]; + static int yAxis[K_CHANNELS]; + if (init) { + for (int i = 0; i < K_CHANNELS; ++i) { + show[i] = false; + yAxis[i] = 0; + } + init = false; + } + ImGui::BulletText("Drag data items from the left column onto the plot or onto a specific y-axis."); + ImGui::BulletText("Redrag data items from the legend onto other y-axes."); + ImGui::BeginGroup(); + if (ImGui::Button("Clear", ImVec2(100, 0))) { + for (int i = 0; i < K_CHANNELS; ++i) { + show[i] = false; + data[i].Data.shrink(0); + data[i].Offset = 0; + } + } + if (ImGui::Button(paused ? "Resume" : "Pause", ImVec2(100,0))) + paused = !paused; + for (int i = 0; i < K_CHANNELS; ++i) { + char label[16]; + sprintf(label, show[i] ? "data_%d (Y%d)" : "data_%d", i, yAxis[i]+1); + ImGui::Selectable(label, false, 0, ImVec2(100, 0)); + if (ImGui::BeginDragDropSource(ImGuiDragDropFlags_None)) { + ImGui::SetDragDropPayload("DND_PLOT", &i, sizeof(int)); + ImGui::TextUnformatted(label); + ImGui::EndDragDropSource(); + } + } + ImGui::EndGroup(); + ImGui::SameLine(); + srand((unsigned int)DEMO_TIME*10000000); + static float t = 0; + if (!paused) { + t += ImGui::GetIO().DeltaTime; + for (int i = 0; i < K_CHANNELS; ++i) { + if (show[i]) + data[i].AddPoint(t, (i+1)*0.1f + RandomRange(-0.01f,0.01f)); + } + } + ImPlot::SetNextPlotLimitsX((double)t - 10, t, paused ? ImGuiCond_Once : ImGuiCond_Always); + if (ImPlot::BeginPlot("##DND", NULL, NULL, ImVec2(-1,0), ImPlotFlags_YAxis2 | ImPlotFlags_YAxis3, ImPlotAxisFlags_NoTickLabels)) { + for (int i = 0; i < K_CHANNELS; ++i) { + if (show[i] && data[i].Data.size() > 0) { + char label[K_CHANNELS]; + sprintf(label, "data_%d", i); + ImPlot::SetPlotYAxis(yAxis[i]); + ImPlot::PlotLine(label, &data[i].Data[0].x, &data[i].Data[0].y, data[i].Data.size(), data[i].Offset, 2 * sizeof(float)); + // allow legend labels to be dragged and dropped + if (ImPlot::BeginLegendDragDropSource(label)) { + ImGui::SetDragDropPayload("DND_PLOT", &i, sizeof(int)); + ImGui::TextUnformatted(label); + ImPlot::EndLegendDragDropSource(); + } + } + } + // make our plot a drag and drop target + if (ImGui::BeginDragDropTarget()) { + if (const ImGuiPayload* payload = ImGui::AcceptDragDropPayload("DND_PLOT")) { + int i = *(int*)payload->Data; + show[i] = true; + yAxis[i] = 0; + // set specific y-axis if hovered + for (int y = 0; y < 3; y++) { + if (ImPlot::IsPlotYAxisHovered(y)) + yAxis[i] = y; + } + } + ImGui::EndDragDropTarget(); + } + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Digital and Analog Signals")) { + static bool paused = false; + #define K_PLOT_DIGITAL_CH_COUNT 4 + #define K_PLOT_ANALOG_CH_COUNT 4 + static ScrollingBuffer dataDigital[K_PLOT_DIGITAL_CH_COUNT]; + static ScrollingBuffer dataAnalog[K_PLOT_ANALOG_CH_COUNT]; + static bool showDigital[K_PLOT_DIGITAL_CH_COUNT]; + static bool showAnalog[K_PLOT_ANALOG_CH_COUNT]; + + ImGui::BulletText("You can plot digital and analog signals on the same plot."); + ImGui::BulletText("Digital signals do not respond to Y drag and zoom, so that"); + ImGui::Indent(); + ImGui::Text("you can drag analog signals over the rising/falling digital edge."); + ImGui::Unindent(); + ImGui::BeginGroup(); + if (ImGui::Button("Clear", ImVec2(100, 0))) { + for (int i = 0; i < K_PLOT_DIGITAL_CH_COUNT; ++i) + showDigital[i] = false; + for (int i = 0; i < K_PLOT_ANALOG_CH_COUNT; ++i) + showAnalog[i] = false; + } + if (ImGui::Button(paused ? "Resume" : "Pause", ImVec2(100,0))) + paused = !paused; + for (int i = 0; i < K_PLOT_DIGITAL_CH_COUNT; ++i) { + char label[32]; + sprintf(label, "digital_%d", i); + ImGui::Checkbox(label, &showDigital[i]); + if (ImGui::BeginDragDropSource(ImGuiDragDropFlags_None)) { + ImGui::SetDragDropPayload("DND_DIGITAL_PLOT", &i, sizeof(int)); + ImGui::TextUnformatted(label); + ImGui::EndDragDropSource(); + } + } + for (int i = 0; i < K_PLOT_ANALOG_CH_COUNT; ++i) { + char label[32]; + sprintf(label, "analog_%d", i); + ImGui::Checkbox(label, &showAnalog[i]); + if (ImGui::BeginDragDropSource(ImGuiDragDropFlags_None)) { + ImGui::SetDragDropPayload("DND_ANALOG_PLOT", &i, sizeof(int)); + ImGui::TextUnformatted(label); + ImGui::EndDragDropSource(); + } + } + ImGui::EndGroup(); + ImGui::SameLine(); + static float t = 0; + if (!paused) { + t += ImGui::GetIO().DeltaTime; + //digital signal values + int i = 0; + if (showDigital[i]) + dataDigital[i].AddPoint(t, sinf(2*t) > 0.45); + i++; + if (showDigital[i]) + dataDigital[i].AddPoint(t, sinf(2*t) < 0.45); + i++; + if (showDigital[i]) + dataDigital[i].AddPoint(t, fmodf(t,5.0f)); + i++; + if (showDigital[i]) + dataDigital[i].AddPoint(t, sinf(2*t) < 0.17); + //Analog signal values + i = 0; + if (showAnalog[i]) + dataAnalog[i].AddPoint(t, sinf(2*t)); + i++; + if (showAnalog[i]) + dataAnalog[i].AddPoint(t, cosf(2*t)); + i++; + if (showAnalog[i]) + dataAnalog[i].AddPoint(t, sinf(2*t) * cosf(2*t)); + i++; + if (showAnalog[i]) + dataAnalog[i].AddPoint(t, sinf(2*t) - cosf(2*t)); + } + ImPlot::SetNextPlotLimitsY(-1, 1); + ImPlot::SetNextPlotLimitsX(t - 10.0, t, paused ? ImGuiCond_Once : ImGuiCond_Always); + if (ImPlot::BeginPlot("##Digital")) { + for (int i = 0; i < K_PLOT_DIGITAL_CH_COUNT; ++i) { + if (showDigital[i] && dataDigital[i].Data.size() > 0) { + char label[32]; + sprintf(label, "digital_%d", i); + ImPlot::PlotDigital(label, &dataDigital[i].Data[0].x, &dataDigital[i].Data[0].y, dataDigital[i].Data.size(), dataDigital[i].Offset, 2 * sizeof(float)); + } + } + for (int i = 0; i < K_PLOT_ANALOG_CH_COUNT; ++i) { + if (showAnalog[i]) { + char label[32]; + sprintf(label, "analog_%d", i); + if (dataAnalog[i].Data.size() > 0) + ImPlot::PlotLine(label, &dataAnalog[i].Data[0].x, &dataAnalog[i].Data[0].y, dataAnalog[i].Data.size(), dataAnalog[i].Offset, 2 * sizeof(float)); + } + } + ImPlot::EndPlot(); + } + if (ImGui::BeginDragDropTarget()) { + const ImGuiPayload* payload = ImGui::AcceptDragDropPayload("DND_DIGITAL_PLOT"); + if (payload) { + int i = *(int*)payload->Data; + showDigital[i] = true; + } + else + { + payload = ImGui::AcceptDragDropPayload("DND_ANALOG_PLOT"); + if (payload) { + int i = *(int*)payload->Data; + showAnalog[i] = true; + } + } + ImGui::EndDragDropTarget(); + } + } + if (ImGui::CollapsingHeader("Tables")) { +#ifdef IMGUI_HAS_TABLE + static ImGuiTableFlags flags = ImGuiTableFlags_BordersOuter | ImGuiTableFlags_BordersV | ImGuiTableFlags_RowBg; + static bool anim = true; + static int offset = 0; + ImGui::BulletText("Plots can be used inside of ImGui tables."); + ImGui::Checkbox("Animate",&anim); + if (anim) + offset = (offset + 1) % 100; + if (ImGui::BeginTable("##table", 3, flags, ImVec2(-1,0))) { + ImGui::TableSetupColumn("Electrode", ImGuiTableColumnFlags_WidthFixed, 75.0f); + ImGui::TableSetupColumn("Voltage", ImGuiTableColumnFlags_WidthFixed, 75.0f); + ImGui::TableSetupColumn("EMG Signal"); + ImGui::TableAutoHeaders(); + ImPlot::PushColormap(ImPlotColormap_Cool); + for (int row = 0; row < 10; row++) { + ImGui::TableNextRow(); + static float data[100]; + srand(row); + for (int i = 0; i < 100; ++i) + data[i] = RandomRange(0.0f,10.0f); + ImGui::TableSetColumnIndex(0); + ImGui::Text("EMG %d", row); + ImGui::TableSetColumnIndex(1); + ImGui::Text("%.3f V", data[offset]); + ImGui::TableSetColumnIndex(2); + ImGui::PushID(row); + MyImPlot::Sparkline("##spark",data,100,0,11.0f,offset,ImPlot::GetColormapColor(row),ImVec2(-1, 35)); + ImGui::PopID(); + } + ImPlot::PopColormap(); + ImGui::EndTable(); + } +#else + ImGui::BulletText("You need to merge the ImGui 'tables' branch for this section."); +#endif + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Offset and Stride")) { + static const int k_circles = 11; + static const int k_points_per = 50; + static const int k_size = 2 * k_points_per * k_circles; + static double interleaved_data[k_size]; + for (int p = 0; p < k_points_per; ++p) { + for (int c = 0; c < k_circles; ++c) { + double r = (double)c / (k_circles - 1) * 0.2 + 0.2; + interleaved_data[p*2*k_circles + 2*c + 0] = 0.5 + r * cos((double)p/k_points_per * 6.28); + interleaved_data[p*2*k_circles + 2*c + 1] = 0.5 + r * sin((double)p/k_points_per * 6.28); + } + } + static int offset = 0; + ImGui::BulletText("Offsetting is useful for realtime plots (see above) and circular buffers."); + ImGui::BulletText("Striding is useful for interleaved data (e.g. audio) or plotting structs."); + ImGui::BulletText("Here, all circle data is stored in a single interleaved buffer:"); + ImGui::BulletText("[c0.x0 c0.y0 ... cn.x0 cn.y0 c0.x1 c0.y1 ... cn.x1 cn.y1 ... cn.xm cn.ym]"); + ImGui::BulletText("The offset value indicates which circle point index is considered the first."); + ImGui::BulletText("Offsets can be negative and/or larger than the actual data count."); + ImGui::SliderInt("Offset", &offset, -2*k_points_per, 2*k_points_per); + if (ImPlot::BeginPlot("##strideoffset")) { + ImPlot::PushColormap(ImPlotColormap_Jet); + char buff[16]; + for (int c = 0; c < k_circles; ++c) { + sprintf(buff, "Circle %d", c); + ImPlot::PlotLine(buff, &interleaved_data[c*2 + 0], &interleaved_data[c*2 + 1], k_points_per, offset, 2*k_circles*sizeof(double)); + } + ImPlot::EndPlot(); + ImPlot::PopColormap(); + } + // offset++; uncomment for animation! + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Custom Data and Getters")) { + ImGui::BulletText("You can plot custom structs using the stride feature."); + ImGui::BulletText("Most plotters can also be passed a function pointer for getting data."); + ImGui::Indent(); + ImGui::BulletText("You can optionally pass user data to be given to your getter function."); + ImGui::BulletText("C++ lambdas can be passed as function pointers as well!"); + ImGui::Unindent(); + + MyImPlot::Vector2f vec2_data[2] = { MyImPlot::Vector2f(0,0), MyImPlot::Vector2f(1,1) }; + + if (ImPlot::BeginPlot("##Custom Data")) { + + // custom structs using stride example: + ImPlot::PlotLine("Vector2f", &vec2_data[0].x, &vec2_data[0].y, 2, 0, sizeof(MyImPlot::Vector2f) /* or sizeof(float) * 2 */); + + // custom getter example 1: + ImPlot::PlotLineG("Spiral", MyImPlot::Spiral, NULL, 1000); + + // custom getter example 2: + static MyImPlot::WaveData data1(0.001, 0.2, 2, 0.75); + static MyImPlot::WaveData data2(0.001, 0.2, 4, 0.25); + ImPlot::PlotLineG("Waves", MyImPlot::SineWave, &data1, 1000); + ImPlot::PlotLineG("Waves", MyImPlot::SawWave, &data2, 1000); + ImPlot::PushStyleVar(ImPlotStyleVar_FillAlpha, 0.25f); + ImPlot::PlotShadedG("Waves", MyImPlot::SineWave, &data1, MyImPlot::SawWave, &data2, 1000); + ImPlot::PopStyleVar(); + + // you can also pass C++ lambdas: + // auto lamda = [](void* data, int idx) { ... return ImPlotPoint(x,y); }; + // ImPlot::PlotLine("My Lambda", lambda, data, 1000); + + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Custom Ticks")) { + static bool custom_ticks = true; + static bool custom_labels = true; + ImGui::Checkbox("Show Custom Ticks", &custom_ticks); + if (custom_ticks) { + ImGui::SameLine(); + ImGui::Checkbox("Show Custom Labels", &custom_labels); + } + double pi = 3.14; + const char* pi_str[] = {"PI"}; + static double yticks[] = {1,3,7,9}; + static const char* ylabels[] = {"One","Three","Seven","Nine"}; + static double yticks_aux[] = {0.2,0.4,0.6}; + static const char* ylabels_aux[] = {"A","B","C","D","E","F"}; + if (custom_ticks) { + ImPlot::SetNextPlotTicksX(&pi,1,custom_labels ? pi_str : NULL, true); + ImPlot::SetNextPlotTicksY(yticks, 4, custom_labels ? ylabels : NULL); + ImPlot::SetNextPlotTicksY(yticks_aux, 3, custom_labels ? ylabels_aux : NULL, false, 1); + ImPlot::SetNextPlotTicksY(0, 1, 6, custom_labels ? ylabels_aux : NULL, false, 2); + } + ImPlot::SetNextPlotLimits(2.5,5,0,10); + if (ImPlot::BeginPlot("Custom Ticks", NULL, NULL, ImVec2(-1,0), ImPlotFlags_YAxis2 | ImPlotFlags_YAxis3)) { + // nothing to see here, just the ticks + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Custom Styles")) { + ImPlot::PushColormap(ImPlotColormap_Deep); + // normally you wouldn't change the entire style each frame + ImPlotStyle backup = ImPlot::GetStyle(); + MyImPlot::StyleSeaborn(); + ImPlot::SetNextPlotLimits(-0.5f, 9.5f, 0, 10); + if (ImPlot::BeginPlot("seaborn style", "x-axis", "y-axis")) { + unsigned int lin[10] = {8,8,9,7,8,8,8,9,7,8}; + unsigned int bar[10] = {1,2,5,3,4,1,2,5,3,4}; + unsigned int dot[10] = {7,6,6,7,8,5,6,5,8,7}; + ImPlot::PlotBars("Bars", bar, 10, 0.5f); + ImPlot::PlotLine("Line", lin, 10); + ImPlot::NextColormapColor(); // skip green + ImPlot::PlotScatter("Scatter", dot, 10); + ImPlot::EndPlot(); + } + ImPlot::GetStyle() = backup; + ImPlot::PopColormap(); + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Custom Rendering")) { + if (ImPlot::BeginPlot("##CustomRend")) { + ImVec2 cntr = ImPlot::PlotToPixels(ImPlotPoint(0.5f, 0.5f)); + ImVec2 rmin = ImPlot::PlotToPixels(ImPlotPoint(0.25f, 0.75f)); + ImVec2 rmax = ImPlot::PlotToPixels(ImPlotPoint(0.75f, 0.25f)); + ImPlot::PushPlotClipRect(); + ImPlot::GetPlotDrawList()->AddCircleFilled(cntr,20,IM_COL32(255,255,0,255),20); + ImPlot::GetPlotDrawList()->AddRect(rmin, rmax, IM_COL32(128,0,255,255)); + ImPlot::PopPlotClipRect(); + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Custom Context Menus")) { + ImGui::BulletText("You can implement legend context menus to inject per-item controls and widgets."); + ImGui::BulletText("Right click the legend label/icon to edit custom item attributes."); + + static float frequency = 0.1f; + static float amplitude = 0.5f; + static ImVec4 color = ImVec4(1,1,0,1); + static float alpha = 1.0f; + static bool line = false; + static float thickness = 1; + static bool markers = false; + static bool shaded = false; + + static float vals[101]; + for (int i = 0; i < 101; ++i) + vals[i] = amplitude * sinf(frequency * i); + + ImPlot::SetNextPlotLimits(0,100,-1,1); + if (ImPlot::BeginPlot("Right Click the Legend")) { + // rendering logic + ImPlot::PushStyleVar(ImPlotStyleVar_FillAlpha, alpha); + if (!line) { + ImPlot::SetNextFillStyle(color); + ImPlot::PlotBars("Right Click Me", vals, 101); + } + else { + if (markers) ImPlot::SetNextMarkerStyle(ImPlotMarker_Circle); + ImPlot::SetNextLineStyle(color, thickness); + ImPlot::PlotLine("Right Click Me", vals, 101); + if (shaded) ImPlot::PlotShaded("Right Click Me",vals,101); + } + ImPlot::PopStyleVar(); + // custom legend context menu + if (ImPlot::BeginLegendPopup("Right Click Me")) { + ImGui::SliderFloat("Frequency",&frequency,0,1,"%0.2f"); + ImGui::SliderFloat("Amplitude",&litude,0,1,"%0.2f"); + ImGui::Separator(); + ImGui::ColorEdit3("Color",&color.x); + ImGui::SliderFloat("Transparency",&alpha,0,1,"%.2f"); + ImGui::Checkbox("Line Plot", &line); + if (line) { + ImGui::SliderFloat("Thickness", &thickness, 0, 5); + ImGui::Checkbox("Markers", &markers); + ImGui::Checkbox("Shaded",&shaded); + } + ImPlot::EndLegendPopup(); + } + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + if (ImGui::CollapsingHeader("Custom Plotters and Tooltips")) { + ImGui::BulletText("You can create custom plotters or extend ImPlot using implot_internal.h."); + double dates[] = {1546300800,1546387200,1546473600,1546560000,1546819200,1546905600,1546992000,1547078400,1547164800,1547424000,1547510400,1547596800,1547683200,1547769600,1547942400,1548028800,1548115200,1548201600,1548288000,1548374400,1548633600,1548720000,1548806400,1548892800,1548979200,1549238400,1549324800,1549411200,1549497600,1549584000,1549843200,1549929600,1550016000,1550102400,1550188800,1550361600,1550448000,1550534400,1550620800,1550707200,1550793600,1551052800,1551139200,1551225600,1551312000,1551398400,1551657600,1551744000,1551830400,1551916800,1552003200,1552262400,1552348800,1552435200,1552521600,1552608000,1552867200,1552953600,1553040000,1553126400,1553212800,1553472000,1553558400,1553644800,1553731200,1553817600,1554076800,1554163200,1554249600,1554336000,1554422400,1554681600,1554768000,1554854400,1554940800,1555027200,1555286400,1555372800,1555459200,1555545600,1555632000,1555891200,1555977600,1556064000,1556150400,1556236800,1556496000,1556582400,1556668800,1556755200,1556841600,1557100800,1557187200,1557273600,1557360000,1557446400,1557705600,1557792000,1557878400,1557964800,1558051200,1558310400,1558396800,1558483200,1558569600,1558656000,1558828800,1558915200,1559001600,1559088000,1559174400,1559260800,1559520000,1559606400,1559692800,1559779200,1559865600,1560124800,1560211200,1560297600,1560384000,1560470400,1560729600,1560816000,1560902400,1560988800,1561075200,1561334400,1561420800,1561507200,1561593600,1561680000,1561939200,1562025600,1562112000,1562198400,1562284800,1562544000,1562630400,1562716800,1562803200,1562889600,1563148800,1563235200,1563321600,1563408000,1563494400,1563753600,1563840000,1563926400,1564012800,1564099200,1564358400,1564444800,1564531200,1564617600,1564704000,1564963200,1565049600,1565136000,1565222400,1565308800,1565568000,1565654400,1565740800,1565827200,1565913600,1566172800,1566259200,1566345600,1566432000,1566518400,1566777600,1566864000,1566950400,1567036800,1567123200,1567296000,1567382400,1567468800,1567555200,1567641600,1567728000,1567987200,1568073600,1568160000,1568246400,1568332800,1568592000,1568678400,1568764800,1568851200,1568937600,1569196800,1569283200,1569369600,1569456000,1569542400,1569801600,1569888000,1569974400,1570060800,1570147200,1570406400,1570492800,1570579200,1570665600,1570752000,1571011200,1571097600,1571184000,1571270400,1571356800,1571616000,1571702400,1571788800,1571875200,1571961600}; + double opens[] = {1284.7,1319.9,1318.7,1328,1317.6,1321.6,1314.3,1325,1319.3,1323.1,1324.7,1321.3,1323.5,1322,1281.3,1281.95,1311.1,1315,1314,1313.1,1331.9,1334.2,1341.3,1350.6,1349.8,1346.4,1343.4,1344.9,1335.6,1337.9,1342.5,1337,1338.6,1337,1340.4,1324.65,1324.35,1349.5,1371.3,1367.9,1351.3,1357.8,1356.1,1356,1347.6,1339.1,1320.6,1311.8,1314,1312.4,1312.3,1323.5,1319.1,1327.2,1332.1,1320.3,1323.1,1328,1330.9,1338,1333,1335.3,1345.2,1341.1,1332.5,1314,1314.4,1310.7,1314,1313.1,1315,1313.7,1320,1326.5,1329.2,1314.2,1312.3,1309.5,1297.4,1293.7,1277.9,1295.8,1295.2,1290.3,1294.2,1298,1306.4,1299.8,1302.3,1297,1289.6,1302,1300.7,1303.5,1300.5,1303.2,1306,1318.7,1315,1314.5,1304.1,1294.7,1293.7,1291.2,1290.2,1300.4,1284.2,1284.25,1301.8,1295.9,1296.2,1304.4,1323.1,1340.9,1341,1348,1351.4,1351.4,1343.5,1342.3,1349,1357.6,1357.1,1354.7,1361.4,1375.2,1403.5,1414.7,1433.2,1438,1423.6,1424.4,1418,1399.5,1435.5,1421.25,1434.1,1412.4,1409.8,1412.2,1433.4,1418.4,1429,1428.8,1420.6,1441,1460.4,1441.7,1438.4,1431,1439.3,1427.4,1431.9,1439.5,1443.7,1425.6,1457.5,1451.2,1481.1,1486.7,1512.1,1515.9,1509.2,1522.3,1513,1526.6,1533.9,1523,1506.3,1518.4,1512.4,1508.8,1545.4,1537.3,1551.8,1549.4,1536.9,1535.25,1537.95,1535.2,1556,1561.4,1525.6,1516.4,1507,1493.9,1504.9,1506.5,1513.1,1506.5,1509.7,1502,1506.8,1521.5,1529.8,1539.8,1510.9,1511.8,1501.7,1478,1485.4,1505.6,1511.6,1518.6,1498.7,1510.9,1510.8,1498.3,1492,1497.7,1484.8,1494.2,1495.6,1495.6,1487.5,1491.1,1495.1,1506.4}; + double highs[] = {1284.75,1320.6,1327,1330.8,1326.8,1321.6,1326,1328,1325.8,1327.1,1326,1326,1323.5,1322.1,1282.7,1282.95,1315.8,1316.3,1314,1333.2,1334.7,1341.7,1353.2,1354.6,1352.2,1346.4,1345.7,1344.9,1340.7,1344.2,1342.7,1342.1,1345.2,1342,1350,1324.95,1330.75,1369.6,1374.3,1368.4,1359.8,1359,1357,1356,1353.4,1340.6,1322.3,1314.1,1316.1,1312.9,1325.7,1323.5,1326.3,1336,1332.1,1330.1,1330.4,1334.7,1341.1,1344.2,1338.8,1348.4,1345.6,1342.8,1334.7,1322.3,1319.3,1314.7,1316.6,1316.4,1315,1325.4,1328.3,1332.2,1329.2,1316.9,1312.3,1309.5,1299.6,1296.9,1277.9,1299.5,1296.2,1298.4,1302.5,1308.7,1306.4,1305.9,1307,1297.2,1301.7,1305,1305.3,1310.2,1307,1308,1319.8,1321.7,1318.7,1316.2,1305.9,1295.8,1293.8,1293.7,1304.2,1302,1285.15,1286.85,1304,1302,1305.2,1323,1344.1,1345.2,1360.1,1355.3,1363.8,1353,1344.7,1353.6,1358,1373.6,1358.2,1369.6,1377.6,1408.9,1425.5,1435.9,1453.7,1438,1426,1439.1,1418,1435,1452.6,1426.65,1437.5,1421.5,1414.1,1433.3,1441.3,1431.4,1433.9,1432.4,1440.8,1462.3,1467,1443.5,1444,1442.9,1447,1437.6,1440.8,1445.7,1447.8,1458.2,1461.9,1481.8,1486.8,1522.7,1521.3,1521.1,1531.5,1546.1,1534.9,1537.7,1538.6,1523.6,1518.8,1518.4,1514.6,1540.3,1565,1554.5,1556.6,1559.8,1541.9,1542.9,1540.05,1558.9,1566.2,1561.9,1536.2,1523.8,1509.1,1506.2,1532.2,1516.6,1519.7,1515,1519.5,1512.1,1524.5,1534.4,1543.3,1543.3,1542.8,1519.5,1507.2,1493.5,1511.4,1525.8,1522.2,1518.8,1515.3,1518,1522.3,1508,1501.5,1503,1495.5,1501.1,1497.9,1498.7,1492.1,1499.4,1506.9,1520.9}; + double lows[] = {1282.85,1315,1318.7,1309.6,1317.6,1312.9,1312.4,1319.1,1319,1321,1318.1,1321.3,1319.9,1312,1280.5,1276.15,1308,1309.9,1308.5,1312.3,1329.3,1333.1,1340.2,1347,1345.9,1338,1340.8,1335,1332,1337.9,1333,1336.8,1333.2,1329.9,1340.4,1323.85,1324.05,1349,1366.3,1351.2,1349.1,1352.4,1350.7,1344.3,1338.9,1316.3,1308.4,1306.9,1309.6,1306.7,1312.3,1315.4,1319,1327.2,1317.2,1320,1323,1328,1323,1327.8,1331.7,1335.3,1336.6,1331.8,1311.4,1310,1309.5,1308,1310.6,1302.8,1306.6,1313.7,1320,1322.8,1311,1312.1,1303.6,1293.9,1293.5,1291,1277.9,1294.1,1286,1289.1,1293.5,1296.9,1298,1299.6,1292.9,1285.1,1288.5,1296.3,1297.2,1298.4,1298.6,1302,1300.3,1312,1310.8,1301.9,1292,1291.1,1286.3,1289.2,1289.9,1297.4,1283.65,1283.25,1292.9,1295.9,1290.8,1304.2,1322.7,1336.1,1341,1343.5,1345.8,1340.3,1335.1,1341.5,1347.6,1352.8,1348.2,1353.7,1356.5,1373.3,1398,1414.7,1427,1416.4,1412.7,1420.1,1396.4,1398.8,1426.6,1412.85,1400.7,1406,1399.8,1404.4,1415.5,1417.2,1421.9,1415,1413.7,1428.1,1434,1435.7,1427.5,1429.4,1423.9,1425.6,1427.5,1434.8,1422.3,1412.1,1442.5,1448.8,1468.2,1484.3,1501.6,1506.2,1498.6,1488.9,1504.5,1518.3,1513.9,1503.3,1503,1506.5,1502.1,1503,1534.8,1535.3,1541.4,1528.6,1525.6,1535.25,1528.15,1528,1542.6,1514.3,1510.7,1505.5,1492.1,1492.9,1496.8,1493.1,1503.4,1500.9,1490.7,1496.3,1505.3,1505.3,1517.9,1507.4,1507.1,1493.3,1470.5,1465,1480.5,1501.7,1501.4,1493.3,1492.1,1505.1,1495.7,1478,1487.1,1480.8,1480.6,1487,1488.3,1484.8,1484,1490.7,1490.4,1503.1}; + double closes[] = {1283.35,1315.3,1326.1,1317.4,1321.5,1317.4,1323.5,1319.2,1321.3,1323.3,1319.7,1325.1,1323.6,1313.8,1282.05,1279.05,1314.2,1315.2,1310.8,1329.1,1334.5,1340.2,1340.5,1350,1347.1,1344.3,1344.6,1339.7,1339.4,1343.7,1337,1338.9,1340.1,1338.7,1346.8,1324.25,1329.55,1369.6,1372.5,1352.4,1357.6,1354.2,1353.4,1346,1341,1323.8,1311.9,1309.1,1312.2,1310.7,1324.3,1315.7,1322.4,1333.8,1319.4,1327.1,1325.8,1330.9,1325.8,1331.6,1336.5,1346.7,1339.2,1334.7,1313.3,1316.5,1312.4,1313.4,1313.3,1312.2,1313.7,1319.9,1326.3,1331.9,1311.3,1313.4,1309.4,1295.2,1294.7,1294.1,1277.9,1295.8,1291.2,1297.4,1297.7,1306.8,1299.4,1303.6,1302.2,1289.9,1299.2,1301.8,1303.6,1299.5,1303.2,1305.3,1319.5,1313.6,1315.1,1303.5,1293,1294.6,1290.4,1291.4,1302.7,1301,1284.15,1284.95,1294.3,1297.9,1304.1,1322.6,1339.3,1340.1,1344.9,1354,1357.4,1340.7,1342.7,1348.2,1355.1,1355.9,1354.2,1362.1,1360.1,1408.3,1411.2,1429.5,1430.1,1426.8,1423.4,1425.1,1400.8,1419.8,1432.9,1423.55,1412.1,1412.2,1412.8,1424.9,1419.3,1424.8,1426.1,1423.6,1435.9,1440.8,1439.4,1439.7,1434.5,1436.5,1427.5,1432.2,1433.3,1441.8,1437.8,1432.4,1457.5,1476.5,1484.2,1519.6,1509.5,1508.5,1517.2,1514.1,1527.8,1531.2,1523.6,1511.6,1515.7,1515.7,1508.5,1537.6,1537.2,1551.8,1549.1,1536.9,1529.4,1538.05,1535.15,1555.9,1560.4,1525.5,1515.5,1511.1,1499.2,1503.2,1507.4,1499.5,1511.5,1513.4,1515.8,1506.2,1515.1,1531.5,1540.2,1512.3,1515.2,1506.4,1472.9,1489,1507.9,1513.8,1512.9,1504.4,1503.9,1512.8,1500.9,1488.7,1497.6,1483.5,1494,1498.3,1494.1,1488.1,1487.5,1495.7,1504.7,1505.3}; + static bool tooltip = true; + ImGui::Checkbox("Show Tooltip", &tooltip); + ImGui::SameLine(); + static ImVec4 bullCol = ImVec4(0.000f, 1.000f, 0.441f, 1.000f); + static ImVec4 bearCol = ImVec4(0.853f, 0.050f, 0.310f, 1.000f); + ImGui::SameLine(); ImGui::ColorEdit4("##Bull", &bullCol.x, ImGuiColorEditFlags_NoInputs); + ImGui::SameLine(); ImGui::ColorEdit4("##Bear", &bearCol.x, ImGuiColorEditFlags_NoInputs); + ImPlot::GetStyle().UseLocalTime = false; + ImPlot::SetNextPlotLimits(1546300800, 1571961600, 1250, 1600); + if (ImPlot::BeginPlot("Candlestick Chart","Day","USD",ImVec2(-1,-1),0,ImPlotAxisFlags_Time)) { + MyImPlot::PlotCandlestick("GOOGL",dates, opens, closes, lows, highs, 218, tooltip, 0.25f, bullCol, bearCol); + ImPlot::EndPlot(); + } + } + //------------------------------------------------------------------------- + ImGui::End(); +} + +} // namespace ImPlot + +namespace MyImPlot { + +ImPlotPoint SineWave(void* data , int idx) { + WaveData* wd = (WaveData*)data; + double x = idx * wd->X; + return ImPlotPoint(x, wd->Offset + wd->Amp * sin(2 * 3.14 * wd->Freq * x)); +} + +ImPlotPoint SawWave(void* data, int idx) { + WaveData* wd = (WaveData*)data; + double x = idx * wd->X; + return ImPlotPoint(x, wd->Offset + wd->Amp * (-2 / 3.14 * atan(cos(3.14 * wd->Freq * x) / sin(3.14 * wd->Freq * x)))); +} + +ImPlotPoint Spiral(void*, int idx) { + float r = 0.9f; // outer radius + float a = 0; // inner radius + float b = 0.05f; // increment per rev + float n = (r - a) / b; // number of revolutions + double th = 2 * n * 3.14; // angle + float Th = float(th * idx / (1000 - 1)); + return ImPlotPoint(0.5f+(a + b*Th / (2.0f * (float) 3.14))*cos(Th), + 0.5f + (a + b*Th / (2.0f * (float)3.14))*sin(Th)); +} + +// Example for Tables section. Generates a quick and simple shaded line plot. See implementation at bottom. +void Sparkline(const char* id, const float* values, int count, float min_v, float max_v, int offset, const ImVec4& col, const ImVec2& size) { + ImPlot::PushStyleVar(ImPlotStyleVar_PlotPadding, ImVec2(0,0)); + ImPlot::SetNextPlotLimits(0, count - 1, min_v, max_v, ImGuiCond_Always); + if (ImPlot::BeginPlot(id,0,0,size,ImPlotFlags_CanvasOnly|ImPlotFlags_NoChild,ImPlotAxisFlags_NoDecorations,ImPlotAxisFlags_NoDecorations)) { + ImPlot::PushStyleColor(ImPlotCol_Line, col); + ImPlot::PlotLine(id, values, count, offset); + ImPlot::PushStyleVar(ImPlotStyleVar_FillAlpha, 0.25f); + ImPlot::PlotShaded(id, values, count, 0, offset); + ImPlot::PopStyleVar(); + ImPlot::PopStyleColor(); + ImPlot::EndPlot(); + } + ImPlot::PopStyleVar(); +} + +void StyleSeaborn() { + + ImPlotStyle& style = ImPlot::GetStyle(); + + ImVec4* colors = style.Colors; + colors[ImPlotCol_Line] = IMPLOT_AUTO_COL; + colors[ImPlotCol_Fill] = IMPLOT_AUTO_COL; + colors[ImPlotCol_MarkerOutline] = IMPLOT_AUTO_COL; + colors[ImPlotCol_MarkerFill] = IMPLOT_AUTO_COL; + colors[ImPlotCol_ErrorBar] = ImVec4(0.00f, 0.00f, 0.00f, 1.00f); + colors[ImPlotCol_FrameBg] = ImVec4(1.00f, 1.00f, 1.00f, 1.00f); + colors[ImPlotCol_PlotBg] = ImVec4(0.92f, 0.92f, 0.95f, 1.00f); + colors[ImPlotCol_PlotBorder] = ImVec4(0.00f, 0.00f, 0.00f, 0.00f); + colors[ImPlotCol_LegendBg] = ImVec4(0.92f, 0.92f, 0.95f, 1.00f); + colors[ImPlotCol_LegendBorder] = ImVec4(0.80f, 0.81f, 0.85f, 1.00f); + colors[ImPlotCol_LegendText] = ImVec4(0.00f, 0.00f, 0.00f, 1.00f); + colors[ImPlotCol_TitleText] = ImVec4(0.00f, 0.00f, 0.00f, 1.00f); + colors[ImPlotCol_InlayText] = ImVec4(0.00f, 0.00f, 0.00f, 1.00f); + colors[ImPlotCol_XAxis] = ImVec4(0.00f, 0.00f, 0.00f, 1.00f); + colors[ImPlotCol_XAxisGrid] = ImVec4(1.00f, 1.00f, 1.00f, 1.00f); + colors[ImPlotCol_YAxis] = ImVec4(0.00f, 0.00f, 0.00f, 1.00f); + colors[ImPlotCol_YAxisGrid] = ImVec4(1.00f, 1.00f, 1.00f, 1.00f); + colors[ImPlotCol_YAxis2] = ImVec4(0.00f, 0.00f, 0.00f, 1.00f); + colors[ImPlotCol_YAxisGrid2] = ImVec4(1.00f, 1.00f, 1.00f, 1.00f); + colors[ImPlotCol_YAxis3] = ImVec4(0.00f, 0.00f, 0.00f, 1.00f); + colors[ImPlotCol_YAxisGrid3] = ImVec4(1.00f, 1.00f, 1.00f, 1.00f); + colors[ImPlotCol_Selection] = ImVec4(1.00f, 0.65f, 0.00f, 1.00f); + colors[ImPlotCol_Query] = ImVec4(0.23f, 0.10f, 0.64f, 1.00f); + colors[ImPlotCol_Crosshairs] = ImVec4(0.23f, 0.10f, 0.64f, 0.50f); + + style.LineWeight = 1.5; + style.Marker = ImPlotMarker_None; + style.MarkerSize = 4; + style.MarkerWeight = 1; + style.FillAlpha = 1.0f; + style.ErrorBarSize = 5; + style.ErrorBarWeight = 1.5f; + style.DigitalBitHeight = 8; + style.DigitalBitGap = 4; + style.PlotBorderSize = 0; + style.MinorAlpha = 1.0f; + style.MajorTickLen = ImVec2(0,0); + style.MinorTickLen = ImVec2(0,0); + style.MajorTickSize = ImVec2(0,0); + style.MinorTickSize = ImVec2(0,0); + style.MajorGridSize = ImVec2(1.2f,1.2f); + style.MinorGridSize = ImVec2(1.2f,1.2f); + style.PlotPadding = ImVec2(12,12); + style.LabelPadding = ImVec2(5,5); + style.LegendPadding = ImVec2(5,5); + style.InfoPadding = ImVec2(5,5); + style.PlotMinSize = ImVec2(300,225); +} + +} // namespaece MyImPlot + +// WARNING: +// +// You can use "implot_internal.h" to build custom plotting fuctions or extend ImPlot. +// However, note that forward compatibility of this file is not guaranteed and the +// internal API is subject to change. At some point we hope to bring more of this +// into the public API and expose the necessary building blocks to fully support +// custom plotters. For now, proceed at your own risk! + +#include "implot_internal.h" + +namespace MyImPlot { + +template +int BinarySearch(const T* arr, int l, int r, T x) { + if (r >= l) { + int mid = l + (r - l) / 2; + if (arr[mid] == x) + return mid; + if (arr[mid] > x) + return BinarySearch(arr, l, mid - 1, x); + return BinarySearch(arr, mid + 1, r, x); + } + return -1; +} + +void PlotCandlestick(const char* label_id, const double* xs, const double* opens, const double* closes, const double* lows, const double* highs, int count, bool tooltip, float width_percent, ImVec4 bullCol, ImVec4 bearCol) { + + // get ImGui window DrawList + ImDrawList* draw_list = ImPlot::GetPlotDrawList(); + // calc real value width + double half_width = count > 1 ? (xs[1] - xs[0]) * width_percent : width_percent; + + // custom tool + if (ImPlot::IsPlotHovered() && tooltip) { + ImPlotPoint mouse = ImPlot::GetPlotMousePos(); + mouse.x = ImPlot::RoundTime(ImPlotTime::FromDouble(mouse.x), ImPlotTimeUnit_Day).ToDouble(); + float tool_l = ImPlot::PlotToPixels(mouse.x - half_width * 1.5, mouse.y).x; + float tool_r = ImPlot::PlotToPixels(mouse.x + half_width * 1.5, mouse.y).x; + float tool_t = ImPlot::GetPlotPos().y; + float tool_b = tool_t + ImPlot::GetPlotSize().y; + ImPlot::PushPlotClipRect(); + draw_list->AddRectFilled(ImVec2(tool_l, tool_t), ImVec2(tool_r, tool_b), IM_COL32(128,128,128,64)); + ImPlot::PopPlotClipRect(); + // find mouse location index + int idx = BinarySearch(xs, 0, count - 1, mouse.x); + // render tool tip (won't be affected by plot clip rect) + if (idx != -1) { + ImGui::BeginTooltip(); + char buff[32]; + ImPlot::FormatTime(ImPlotTime::FromDouble(xs[idx]),buff,32,ImPlotTimeFmt_DayMoYr); + ImGui::Text("Day: %s", buff); + ImGui::Text("Open: $%.2f", opens[idx]); + ImGui::Text("Close: $%.2f", closes[idx]); + ImGui::Text("Low: $%.2f", lows[idx]); + ImGui::Text("High: $%.2f", highs[idx]); + ImGui::EndTooltip(); + } + } + + // begin plot item + if (ImPlot::BeginItem(label_id)) { + // override legend icon color + ImPlot::GetCurrentItem()->Color = ImVec4(0.25f,0.25f,0.25f,1); + // fit data if requested + if (ImPlot::FitThisFrame()) { + for (int i = 0; i < count; ++i) { + ImPlot::FitPoint(ImPlotPoint(xs[i], lows[i])); + ImPlot::FitPoint(ImPlotPoint(xs[i], highs[i])); + } + } + // render data + for (int i = 0; i < count; ++i) { + ImVec2 open_pos = ImPlot::PlotToPixels(xs[i] - half_width, opens[i]); + ImVec2 close_pos = ImPlot::PlotToPixels(xs[i] + half_width, closes[i]); + ImVec2 low_pos = ImPlot::PlotToPixels(xs[i], lows[i]); + ImVec2 high_pos = ImPlot::PlotToPixels(xs[i], highs[i]); + ImU32 color = ImGui::GetColorU32(opens[i] > closes[i] ? bearCol : bullCol); + draw_list->AddLine(low_pos, high_pos, color); + draw_list->AddRectFilled(open_pos, close_pos, color); + } + + // end plot item + ImPlot::EndItem(); + } +} + +} // namespace MyImplot + +namespace ImPlot { + +//----------------------------------------------------------------------------- +// BENCHMARK +//----------------------------------------------------------------------------- + +struct BenchData { + BenchData() { + float y = RandomRange(0.0f,1.0f); + Data = new float[1000]; + for (int i = 0; i < 1000; ++i) { + Data[i] = y + RandomRange(-0.01f,0.01f); + } + Col = ImVec4(RandomRange(0.0f,1.0f),RandomRange(0.0f,1.0f),RandomRange(0.0f,1.0f),0.5f); + } + ~BenchData() { delete[] Data; } + float* Data; + ImVec4 Col; +}; + +enum BenchMode { + Line = 0, + Shaded = 1, + Scatter = 2, + Bars = 3 +}; + +struct BenchRecord { + int Mode; + bool AA; + ImVector Data; +}; + +void ShowBenchmarkTool() { + static const int max_items = 500; + static BenchData items[max_items]; + static bool running = false; + static int frames = 60; + static int L = 0; + static int F = 0; + static double t1, t2; + static int mode = BenchMode::Line; + const char* names[] = {"Line","Shaded","Scatter","Bars"}; + + static ImVector records; + + if (running) { + F++; + if (F == frames) { + t2 = ImGui::GetTime(); + records.back().Data.push_back(ImPlotPoint(L, frames / (t2 - t1))); + L += 5; + F = 0; + t1 = ImGui::GetTime(); + } + if (L > max_items) { + running = false; + L = max_items; + } + } + + ImGui::Text("ImDrawIdx: %d-bit", (int)(sizeof(ImDrawIdx) * 8)); + ImGui::Text("ImGuiBackendFlags_RendererHasVtxOffset: %s", (ImGui::GetIO().BackendFlags & ImGuiBackendFlags_RendererHasVtxOffset) ? "True" : "False"); + ImGui::Text("%.2f FPS", ImGui::GetIO().Framerate); + + ImGui::Separator(); + + bool was_running = running; + if (was_running) { + ImGui::PushItemFlag(ImGuiItemFlags_Disabled, true); + ImGui::PushStyleVar(ImGuiStyleVar_Alpha, ImGui::GetStyle().Alpha * 0.25f); + } + if (ImGui::Button("Benchmark")) { + running = true; + L = F = 0; + records.push_back(BenchRecord()); + records.back().Data.reserve(max_items+1); + records.back().Mode = mode; + records.back().AA = ImPlot::GetStyle().AntiAliasedLines; + t1 = ImGui::GetTime(); + } + ImGui::SameLine(); + ImGui::SetNextItemWidth(200); + ImGui::Combo("##Mode",&mode,names,4); + ImGui::SameLine(); + + ImGui::Checkbox("Anti-Aliased Lines", &ImPlot::GetStyle().AntiAliasedLines); + if (was_running) { ImGui::PopItemFlag(); ImGui::PopStyleVar(); } + + ImGui::ProgressBar((float)L / (float)(max_items - 1)); + + ImPlot::SetNextPlotLimits(0,1000,0,1,ImGuiCond_Always); + if (ImPlot::BeginPlot("##Bench",NULL,NULL,ImVec2(-1,0),ImPlotFlags_NoChild | ImPlotFlags_CanvasOnly,ImPlotAxisFlags_NoDecorations,ImPlotAxisFlags_NoDecorations)) { + if (running) { + if (mode == BenchMode::Line) { + for (int i = 0; i < L; ++i) { + ImGui::PushID(i); + ImPlot::SetNextLineStyle(items[i].Col); + ImPlot::PlotLine("##item", items[i].Data, 1000); + ImGui::PopID(); + } + } + else if (mode == BenchMode::Shaded) { + for (int i = 0; i < L; ++i) { + ImGui::PushID(i); + ImPlot::SetNextFillStyle(items[i].Col,0.5f); + ImPlot::PlotShaded("##item", items[i].Data, 1000); + ImGui::PopID(); + } + } + else if (mode == BenchMode::Scatter) { + for (int i = 0; i < L; ++i) { + ImGui::PushID(i); + ImPlot::SetNextLineStyle(items[i].Col); + ImPlot::PlotScatter("##item", items[i].Data, 1000); + ImGui::PopID(); + } + } + else if (mode == BenchMode::Bars) { + for (int i = 0; i < L; ++i) { + ImGui::PushID(i); + ImPlot::SetNextFillStyle(items[i].Col,0.5f); + ImPlot::PlotBars("##item", items[i].Data, 1000); + ImGui::PopID(); + } + } + } + ImPlot::EndPlot(); + } + + ImPlot::SetNextPlotLimits(0,500,0,500,ImGuiCond_Always); + static char buffer[64]; + if (ImPlot::BeginPlot("##Stats", "Items (1,000 pts each)", "Framerate (Hz)", ImVec2(-1,0), ImPlotFlags_NoChild)) { + for (int run = 0; run < records.size(); ++run) { + sprintf(buffer, "B%d-%s%s", run + 1, names[records[run].Mode], records[run].AA ? "-AA" : ""); + ImPlot::PlotLine(buffer, records[run].Data.Data, records[run].Data.Size); + } + ImPlot::EndPlot(); + } +} + +} diff --git a/src/external/imgui/imgui_monado/imgui_monado.cpp b/src/external/imgui/imgui_monado/imgui_monado.cpp index 38cf7e335..8f4b1ce59 100644 --- a/src/external/imgui/imgui_monado/imgui_monado.cpp +++ b/src/external/imgui/imgui_monado/imgui_monado.cpp @@ -15,6 +15,8 @@ #endif #include "../imgui/imgui_internal.h" +#include + #include "cimgui_monado.h" using namespace ImGui; @@ -41,20 +43,23 @@ static void _draw_line(ImGuiWindow *window, int values_count, float scale_min, ImGui::RenderText(text_pos, text); ImGui::PopStyleColor(1); } + static void _draw_grid(ImGuiWindow *window, int values_count, float scale_min, float scale_max, float reference_timing, const char *unit, const ImRect inner_bb, ImVec2 frame_size) { - ImVec4 target_color = ImVec4(1.0f, 1.0f, 0.0f, .75f); + const ImVec4 target_color{1.0f, 1.0f, 0.0f, .75f}; _draw_line(window, values_count, scale_min, scale_max, reference_timing, unit, inner_bb, frame_size, GetColorU32(target_color)); - ImVec4 passive_color = ImVec4(0.35f, 0.35f, 0.35f, 1.00f); + const ImVec4 passive_color{0.35f, 0.35f, 0.35f, 1.00f}; // always draw ~5 lines - float step = (scale_max - scale_min) / 5.; - for (float i = scale_min; i < scale_max + step; i += step) { - _draw_line(window, values_count, scale_min, scale_max, i, unit, inner_bb, + const uint8_t max_lines = 5; + float step = (scale_max - scale_min) / (float)max_lines; + for (uint8_t i = 0; i < max_lines; ++i) { + float val = scale_min + step * (float)i; + _draw_line(window, values_count, scale_min, scale_max, val, unit, inner_bb, frame_size, GetColorU32(passive_color)); } } @@ -103,14 +108,16 @@ static void PlotTimings(const char *label, float scale_max = reference_timing + range; if (dynamic_rescale) { - if (v_max > scale_max) + if (v_max > scale_max) { scale_max = v_max; - scale_max = ((int)(scale_max / 10 + 1)) * 10; + } + scale_max = (floorf(scale_max / 10 + 1)) * 10; if (center_reference_timing) { - if (v_min < scale_min) + if (v_min < scale_min) { scale_min = v_min; - scale_min = ((int)(scale_min / 10)) * 10; + } + scale_min = (floorf(scale_min / 10)) * 10; // make sure reference timing stays centered float lower_range = reference_timing - scale_min; diff --git a/src/external/jnipp/jnipp.cpp b/src/external/jnipp/jnipp.cpp index cc9941be1..a250b76cd 100644 --- a/src/external/jnipp/jnipp.cpp +++ b/src/external/jnipp/jnipp.cpp @@ -353,20 +353,20 @@ namespace jni return _handle == nullptr || env()->IsSameObject(_handle, nullptr); } - template <> void Object::callMethod(method_t method, internal::value_t* args) const + void Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { env()->CallVoidMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); } - template <> bool Object::callMethod(method_t method, internal::value_t* args) const + bool Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallBooleanMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return result != 0; } - template <> bool Object::get(field_t field) const + bool Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return env()->GetBooleanField(_handle, field) != 0; } @@ -376,122 +376,122 @@ namespace jni env()->SetBooleanField(_handle, field, value); } - template <> byte_t Object::callMethod(method_t method, internal::value_t* args) const + byte_t Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallByteMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return result; } - template <> wchar_t Object::callMethod(method_t method, internal::value_t* args) const + wchar_t Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallCharMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return result; } - template <> short Object::callMethod(method_t method, internal::value_t* args) const + short Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallShortMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return result; } - template <> int Object::callMethod(method_t method, internal::value_t* args) const + int Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallIntMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return result; } - template <> long long Object::callMethod(method_t method, internal::value_t* args) const + long long Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallLongMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return result; } - template <> float Object::callMethod(method_t method, internal::value_t* args) const + float Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallFloatMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return result; } - template <> double Object::callMethod(method_t method, internal::value_t* args) const + double Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallDoubleMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return result; } - template <> std::string Object::callMethod(method_t method, internal::value_t* args) const + std::string Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallObjectMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return toString(result); } - template <> std::wstring Object::callMethod(method_t method, internal::value_t* args) const + std::wstring Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallObjectMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return toWString(result); } - template <> jni::Object Object::callMethod(method_t method, internal::value_t* args) const + jni::Object Object::callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const { auto result = env()->CallObjectMethodA(_handle, method, (jvalue*) args); handleJavaExceptions(); return Object(result, DeleteLocalInput); } - template <> byte_t Object::get(field_t field) const + byte_t Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return env()->GetByteField(_handle, field); } - template <> wchar_t Object::get(field_t field) const + wchar_t Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return env()->GetCharField(_handle, field); } - template <> short Object::get(field_t field) const + short Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return env()->GetShortField(_handle, field); } - template <> int Object::get(field_t field) const + int Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return env()->GetIntField(_handle, field); } - template <> long long Object::get(field_t field) const + long long Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return env()->GetLongField(_handle, field); } - template <> float Object::get(field_t field) const + float Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return env()->GetFloatField(_handle, field); } - template <> double Object::get(field_t field) const + double Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return env()->GetDoubleField(_handle, field); } - template <> std::string Object::get(field_t field) const + std::string Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return toString(env()->GetObjectField(_handle, field)); } - template <> std::wstring Object::get(field_t field) const + std::wstring Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return toWString(env()->GetObjectField(_handle, field)); } - template <> Object Object::get(field_t field) const + Object Object::getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const { return Object(env()->GetObjectField(_handle, field), DeleteLocalInput); } @@ -1480,6 +1480,10 @@ namespace jni ((jvalue*) v)->l = env()->NewStringUTF(a); } + void valueArg(value_t* v, std::nullptr_t) + { + ((jvalue*) v)->l = nullptr; + } template <> void cleanupArg(value_t* v) { diff --git a/src/external/jnipp/jnipp.h b/src/external/jnipp/jnipp.h index 53e12719e..70515c979 100644 --- a/src/external/jnipp/jnipp.h +++ b/src/external/jnipp/jnipp.h @@ -134,6 +134,7 @@ namespace jni void valueArg(value_t* v, const char* a); void valueArg(value_t* v, const std::wstring& a); void valueArg(value_t* v, const wchar_t* a); + void valueArg(value_t* v, std::nullptr_t); inline void args(value_t*) {} @@ -175,6 +176,11 @@ namespace jni }; long getArrayLength(jarray array); + + template + struct ReturnTypeWrapper{ + using type = T; + }; } /** @@ -281,7 +287,7 @@ namespace jni \return The method's return value. */ template - TReturn call(method_t method) const { return callMethod(method, nullptr); } + TReturn call(method_t method) const { return callMethod(method, nullptr, internal::ReturnTypeWrapper{}); } /** Calls the method on this Object with the given name, and no arguments. @@ -311,7 +317,7 @@ namespace jni template TReturn call(method_t method, const TArgs&... args) const { internal::ArgArray transform(args...); - return callMethod(method, transform.values); + return callMethod(method, transform.values, internal::ReturnTypeWrapper{}); } /** @@ -341,7 +347,11 @@ namespace jni \return The field's value. */ template - TType get(field_t field) const; + TType get(field_t field) const { + // If you get a compile error here, then you've asked for a type + // we don't know how to get from JNI directly. + return getFieldValue(field, internal::ReturnTypeWrapper{}); + } /** Gets a field value from this Object. The field must belong to the @@ -409,7 +419,33 @@ namespace jni method_t getMethod(const char* name, const char* signature) const; method_t getMethod(const char* nameAndSignature) const; field_t getField(const char* name, const char* signature) const; - template TType callMethod(method_t method, internal::value_t* values) const; + + void callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + bool callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + byte_t callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + wchar_t callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + short callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + int callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + long long callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + float callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + double callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + std::string callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + std::wstring callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + jni::Object callMethod(method_t method, internal::value_t* args, internal::ReturnTypeWrapper const&) const; + + + void getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + bool getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + byte_t getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + wchar_t getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + short getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + int getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + long long getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + float getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + double getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + std::string getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + std::wstring getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; + jni::Object getFieldValue(field_t field, internal::ReturnTypeWrapper const&) const; // Instance Variables jobject _handle; diff --git a/src/external/meson.build b/src/external/meson.build index 045ecc529..f71e6efb7 100644 --- a/src/external/meson.build +++ b/src/external/meson.build @@ -9,3 +9,12 @@ hungarian_include = include_directories('hungarian') imgui_include = include_directories('imgui') openxr_include = include_directories('openxr_includes') catch2_include = include_directories('Catch2') +stb_include = include_directories('stb') +slam_tracker_include = include_directories('slam_tracker') + +if slam.found() + external_slam = declare_dependency( + include_directories: [slam_tracker_include], + dependencies: [slam], + ) +endif diff --git a/src/external/openxr_includes/openxr/loader_interfaces.h b/src/external/openxr_includes/openxr/loader_interfaces.h index bf0a4ea16..695f4c089 100644 --- a/src/external/openxr_includes/openxr/loader_interfaces.h +++ b/src/external/openxr_includes/openxr/loader_interfaces.h @@ -1,22 +1,10 @@ -// Copyright (c) 2017-2020 The Khronos Group Inc. +// Copyright (c) 2017-2021, The Khronos Group Inc. // Copyright (c) 2017 Valve Corporation // Copyright (c) 2017 LunarG, Inc. // -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: Apache-2.0 OR MIT // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Mark Young +// Initial Author: Mark Young // #pragma once diff --git a/src/external/openxr_includes/openxr/openxr.h b/src/external/openxr_includes/openxr/openxr.h index 04595ee14..399fa34c9 100644 --- a/src/external/openxr_includes/openxr/openxr.h +++ b/src/external/openxr_includes/openxr/openxr.h @@ -2,7 +2,7 @@ #define OPENXR_H_ 1 /* -** Copyright (c) 2017-2020 The Khronos Group Inc. +** Copyright (c) 2017-2021, The Khronos Group Inc. ** ** SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -25,7 +25,7 @@ extern "C" { ((((major) & 0xffffULL) << 48) | (((minor) & 0xffffULL) << 32) | ((patch) & 0xffffffffULL)) // OpenXR current version number. -#define XR_CURRENT_API_VERSION XR_MAKE_VERSION(1, 0, 13) +#define XR_CURRENT_API_VERSION XR_MAKE_VERSION(1, 0, 19) #define XR_VERSION_MAJOR(version) (uint16_t)(((uint64_t)(version) >> 48)& 0xffffULL) #define XR_VERSION_MINOR(version) (uint16_t)(((uint64_t)(version) >> 32) & 0xffffULL) @@ -181,13 +181,23 @@ typedef enum XrResult { XR_ERROR_LOCALIZED_NAME_DUPLICATED = -48, XR_ERROR_LOCALIZED_NAME_INVALID = -49, XR_ERROR_GRAPHICS_REQUIREMENTS_CALL_MISSING = -50, + XR_ERROR_RUNTIME_UNAVAILABLE = -51, XR_ERROR_ANDROID_THREAD_SETTINGS_ID_INVALID_KHR = -1000003000, XR_ERROR_ANDROID_THREAD_SETTINGS_FAILURE_KHR = -1000003001, XR_ERROR_CREATE_SPATIAL_ANCHOR_FAILED_MSFT = -1000039001, XR_ERROR_SECONDARY_VIEW_CONFIGURATION_TYPE_NOT_ENABLED_MSFT = -1000053000, XR_ERROR_CONTROLLER_MODEL_KEY_INVALID_MSFT = -1000055000, + XR_ERROR_REPROJECTION_MODE_UNSUPPORTED_MSFT = -1000066000, + XR_ERROR_COMPUTE_NEW_SCENE_NOT_COMPLETED_MSFT = -1000097000, + XR_ERROR_SCENE_COMPONENT_ID_INVALID_MSFT = -1000097001, + XR_ERROR_SCENE_COMPONENT_TYPE_MISMATCH_MSFT = -1000097002, + XR_ERROR_SCENE_MESH_BUFFER_ID_INVALID_MSFT = -1000097003, + XR_ERROR_SCENE_COMPUTE_FEATURE_INCOMPATIBLE_MSFT = -1000097004, + XR_ERROR_SCENE_COMPUTE_CONSISTENCY_MISMATCH_MSFT = -1000097005, XR_ERROR_DISPLAY_REFRESH_RATE_UNSUPPORTED_FB = -1000101000, XR_ERROR_COLOR_SPACE_UNSUPPORTED_FB = -1000108000, + XR_ERROR_SPATIAL_ANCHOR_NAME_NOT_FOUND_MSFT = -1000142001, + XR_ERROR_SPATIAL_ANCHOR_NAME_INVALID_MSFT = -1000142002, XR_RESULT_MAX_ENUM = 0x7FFFFFFF } XrResult; @@ -280,6 +290,8 @@ typedef enum XrStructureType { XR_TYPE_COMPOSITION_LAYER_COLOR_SCALE_BIAS_KHR = 1000034000, XR_TYPE_SPATIAL_ANCHOR_CREATE_INFO_MSFT = 1000039000, XR_TYPE_SPATIAL_ANCHOR_SPACE_CREATE_INFO_MSFT = 1000039001, + XR_TYPE_COMPOSITION_LAYER_IMAGE_LAYOUT_FB = 1000040000, + XR_TYPE_COMPOSITION_LAYER_ALPHA_BLEND_FB = 1000041001, XR_TYPE_VIEW_CONFIGURATION_DEPTH_RANGE_EXT = 1000046000, XR_TYPE_GRAPHICS_BINDING_EGL_MNDX = 1000048004, XR_TYPE_SPATIAL_GRAPH_NODE_SPACE_CREATE_INFO_MSFT = 1000049000, @@ -306,15 +318,61 @@ typedef enum XrStructureType { XR_TYPE_CONTROLLER_MODEL_STATE_MSFT = 1000055004, XR_TYPE_VIEW_CONFIGURATION_VIEW_FOV_EPIC = 1000059000, XR_TYPE_HOLOGRAPHIC_WINDOW_ATTACHMENT_MSFT = 1000063000, + XR_TYPE_COMPOSITION_LAYER_REPROJECTION_INFO_MSFT = 1000066000, + XR_TYPE_COMPOSITION_LAYER_REPROJECTION_PLANE_OVERRIDE_MSFT = 1000066001, + XR_TYPE_ANDROID_SURFACE_SWAPCHAIN_CREATE_INFO_FB = 1000070000, + XR_TYPE_COMPOSITION_LAYER_SECURE_CONTENT_FB = 1000072000, XR_TYPE_INTERACTION_PROFILE_ANALOG_THRESHOLD_VALVE = 1000079000, + XR_TYPE_HAND_JOINTS_MOTION_RANGE_INFO_EXT = 1000080000, XR_TYPE_LOADER_INIT_INFO_ANDROID_KHR = 1000089000, XR_TYPE_VULKAN_INSTANCE_CREATE_INFO_KHR = 1000090000, XR_TYPE_VULKAN_DEVICE_CREATE_INFO_KHR = 1000090001, XR_TYPE_VULKAN_GRAPHICS_DEVICE_GET_INFO_KHR = 1000090003, XR_TYPE_COMPOSITION_LAYER_EQUIRECT2_KHR = 1000091000, + XR_TYPE_SCENE_OBSERVER_CREATE_INFO_MSFT = 1000097000, + XR_TYPE_SCENE_CREATE_INFO_MSFT = 1000097001, + XR_TYPE_NEW_SCENE_COMPUTE_INFO_MSFT = 1000097002, + XR_TYPE_VISUAL_MESH_COMPUTE_LOD_INFO_MSFT = 1000097003, + XR_TYPE_SCENE_COMPONENTS_MSFT = 1000097004, + XR_TYPE_SCENE_COMPONENTS_GET_INFO_MSFT = 1000097005, + XR_TYPE_SCENE_COMPONENT_LOCATIONS_MSFT = 1000097006, + XR_TYPE_SCENE_COMPONENTS_LOCATE_INFO_MSFT = 1000097007, + XR_TYPE_SCENE_OBJECTS_MSFT = 1000097008, + XR_TYPE_SCENE_COMPONENT_PARENT_FILTER_INFO_MSFT = 1000097009, + XR_TYPE_SCENE_OBJECT_TYPES_FILTER_INFO_MSFT = 1000097010, + XR_TYPE_SCENE_PLANES_MSFT = 1000097011, + XR_TYPE_SCENE_PLANE_ALIGNMENT_FILTER_INFO_MSFT = 1000097012, + XR_TYPE_SCENE_MESHES_MSFT = 1000097013, + XR_TYPE_SCENE_MESH_BUFFERS_GET_INFO_MSFT = 1000097014, + XR_TYPE_SCENE_MESH_BUFFERS_MSFT = 1000097015, + XR_TYPE_SCENE_MESH_VERTEX_BUFFER_MSFT = 1000097016, + XR_TYPE_SCENE_MESH_INDICES_UINT32_MSFT = 1000097017, + XR_TYPE_SCENE_MESH_INDICES_UINT16_MSFT = 1000097018, + XR_TYPE_SERIALIZED_SCENE_FRAGMENT_DATA_GET_INFO_MSFT = 1000098000, + XR_TYPE_SCENE_DESERIALIZE_INFO_MSFT = 1000098001, XR_TYPE_EVENT_DATA_DISPLAY_REFRESH_RATE_CHANGED_FB = 1000101000, XR_TYPE_SYSTEM_COLOR_SPACE_PROPERTIES_FB = 1000108000, + XR_TYPE_HAND_TRACKING_MESH_FB = 1000110001, + XR_TYPE_HAND_TRACKING_SCALE_FB = 1000110003, + XR_TYPE_HAND_TRACKING_AIM_STATE_FB = 1000111001, + XR_TYPE_HAND_TRACKING_CAPSULES_STATE_FB = 1000112000, + XR_TYPE_FOVEATION_PROFILE_CREATE_INFO_FB = 1000114000, + XR_TYPE_SWAPCHAIN_CREATE_INFO_FOVEATION_FB = 1000114001, + XR_TYPE_SWAPCHAIN_STATE_FOVEATION_FB = 1000114002, + XR_TYPE_FOVEATION_LEVEL_PROFILE_CREATE_INFO_FB = 1000115000, XR_TYPE_BINDING_MODIFICATIONS_KHR = 1000120000, + XR_TYPE_VIEW_LOCATE_FOVEATED_RENDERING_VARJO = 1000121000, + XR_TYPE_FOVEATED_VIEW_CONFIGURATION_VIEW_VARJO = 1000121001, + XR_TYPE_SYSTEM_FOVEATED_RENDERING_PROPERTIES_VARJO = 1000121002, + XR_TYPE_COMPOSITION_LAYER_DEPTH_TEST_VARJO = 1000122000, + XR_TYPE_SPATIAL_ANCHOR_PERSISTENCE_INFO_MSFT = 1000142000, + XR_TYPE_SPATIAL_ANCHOR_FROM_PERSISTED_ANCHOR_CREATE_INFO_MSFT = 1000142001, + XR_TYPE_SWAPCHAIN_IMAGE_FOVEATION_VULKAN_FB = 1000160000, + XR_TYPE_SWAPCHAIN_STATE_ANDROID_SURFACE_DIMENSIONS_FB = 1000161000, + XR_TYPE_SWAPCHAIN_STATE_SAMPLER_OPENGL_ES_FB = 1000162000, + XR_TYPE_SWAPCHAIN_STATE_SAMPLER_VULKAN_FB = 1000163000, + XR_TYPE_COMPOSITION_LAYER_SPACE_WARP_INFO_FB = 1000171000, + XR_TYPE_SYSTEM_SPACE_WARP_PROPERTIES_FB = 1000171001, XR_TYPE_GRAPHICS_BINDING_VULKAN2_KHR = XR_TYPE_GRAPHICS_BINDING_VULKAN_KHR, XR_TYPE_SWAPCHAIN_IMAGE_VULKAN2_KHR = XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR, XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN2_KHR = XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN_KHR, @@ -347,6 +405,7 @@ typedef enum XrReferenceSpaceType { XR_REFERENCE_SPACE_TYPE_LOCAL = 2, XR_REFERENCE_SPACE_TYPE_STAGE = 3, XR_REFERENCE_SPACE_TYPE_UNBOUNDED_MSFT = 1000038000, + XR_REFERENCE_SPACE_TYPE_COMBINED_EYE_VARJO = 1000121000, XR_REFERENCE_SPACE_TYPE_MAX_ENUM = 0x7FFFFFFF } XrReferenceSpaceType; @@ -390,6 +449,10 @@ typedef enum XrObjectType { XR_OBJECT_TYPE_DEBUG_UTILS_MESSENGER_EXT = 1000019000, XR_OBJECT_TYPE_SPATIAL_ANCHOR_MSFT = 1000039000, XR_OBJECT_TYPE_HAND_TRACKER_EXT = 1000051000, + XR_OBJECT_TYPE_SCENE_OBSERVER_MSFT = 1000097000, + XR_OBJECT_TYPE_SCENE_MSFT = 1000097001, + XR_OBJECT_TYPE_FOVEATION_PROFILE_FB = 1000114000, + XR_OBJECT_TYPE_SPATIAL_ANCHOR_STORE_CONNECTION_MSFT = 1000142000, XR_OBJECT_TYPE_MAX_ENUM = 0x7FFFFFFF } XrObjectType; typedef XrFlags64 XrInstanceCreateFlags; @@ -431,6 +494,7 @@ static const XrSwapchainUsageFlags XR_SWAPCHAIN_USAGE_TRANSFER_DST_BIT = 0x00000 static const XrSwapchainUsageFlags XR_SWAPCHAIN_USAGE_SAMPLED_BIT = 0x00000020; static const XrSwapchainUsageFlags XR_SWAPCHAIN_USAGE_MUTABLE_FORMAT_BIT = 0x00000040; static const XrSwapchainUsageFlags XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_MND = 0x00000080; +static const XrSwapchainUsageFlags XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_KHR = 0x00000080; // alias of XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_MND typedef XrFlags64 XrCompositionLayerFlags; @@ -543,6 +607,7 @@ typedef struct XrVector3f { float z; } XrVector3f; +// XrSpaceVelocity extends XrSpaceLocation typedef struct XrSpaceVelocity { XrStructureType type; void* XR_MAY_ALIAS next; @@ -1295,7 +1360,7 @@ XRAPI_ATTR XrResult XRAPI_CALL xrApplyHapticFeedback( XRAPI_ATTR XrResult XRAPI_CALL xrStopHapticFeedback( XrSession session, const XrHapticActionInfo* hapticActionInfo); -#endif +#endif /* !XR_NO_PROTOTYPES */ #define XR_KHR_composition_layer_cube 1 @@ -1317,6 +1382,7 @@ typedef struct XrCompositionLayerCubeKHR { #define XR_KHR_composition_layer_depth 1 #define XR_KHR_composition_layer_depth_SPEC_VERSION 5 #define XR_KHR_COMPOSITION_LAYER_DEPTH_EXTENSION_NAME "XR_KHR_composition_layer_depth" +// XrCompositionLayerDepthInfoKHR extends XrCompositionLayerProjectionView typedef struct XrCompositionLayerDepthInfoKHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -1397,18 +1463,21 @@ typedef struct XrEventDataVisibilityMaskChangedKHR { typedef XrResult (XRAPI_PTR *PFN_xrGetVisibilityMaskKHR)(XrSession session, XrViewConfigurationType viewConfigurationType, uint32_t viewIndex, XrVisibilityMaskTypeKHR visibilityMaskType, XrVisibilityMaskKHR* visibilityMask); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrGetVisibilityMaskKHR( XrSession session, XrViewConfigurationType viewConfigurationType, uint32_t viewIndex, XrVisibilityMaskTypeKHR visibilityMaskType, XrVisibilityMaskKHR* visibilityMask); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_KHR_composition_layer_color_scale_bias 1 #define XR_KHR_composition_layer_color_scale_bias_SPEC_VERSION 5 #define XR_KHR_COMPOSITION_LAYER_COLOR_SCALE_BIAS_EXTENSION_NAME "XR_KHR_composition_layer_color_scale_bias" +// XrCompositionLayerColorScaleBiasKHR extends XrCompositionLayerBaseHeader typedef struct XrCompositionLayerColorScaleBiasKHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -1429,9 +1498,11 @@ typedef struct XR_MAY_ALIAS XrLoaderInitInfoBaseHeaderKHR { typedef XrResult (XRAPI_PTR *PFN_xrInitializeLoaderKHR)(const XrLoaderInitInfoBaseHeaderKHR* loaderInitInfo); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrInitializeLoaderKHR( const XrLoaderInitInfoBaseHeaderKHR* loaderInitInfo); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_KHR_composition_layer_equirect2 1 @@ -1456,11 +1527,12 @@ typedef struct XrCompositionLayerEquirect2KHR { #define XR_KHR_binding_modification 1 #define XR_KHR_binding_modification_SPEC_VERSION 1 #define XR_KHR_BINDING_MODIFICATION_EXTENSION_NAME "XR_KHR_binding_modification" -typedef struct XrBindingModificationBaseHeaderKHR { +typedef struct XR_MAY_ALIAS XrBindingModificationBaseHeaderKHR { XrStructureType type; const void* XR_MAY_ALIAS next; } XrBindingModificationBaseHeaderKHR; +// XrBindingModificationsKHR extends XrInteractionProfileSuggestedBinding typedef struct XrBindingModificationsKHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -1470,8 +1542,13 @@ typedef struct XrBindingModificationsKHR { +#define XR_KHR_swapchain_usage_input_attachment_bit 1 +#define XR_KHR_swapchain_usage_input_attachment_bit_SPEC_VERSION 3 +#define XR_KHR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_EXTENSION_NAME "XR_KHR_swapchain_usage_input_attachment_bit" + + #define XR_EXT_performance_settings 1 -#define XR_EXT_performance_settings_SPEC_VERSION 1 +#define XR_EXT_performance_settings_SPEC_VERSION 3 #define XR_EXT_PERFORMANCE_SETTINGS_EXTENSION_NAME "XR_EXT_performance_settings" typedef enum XrPerfSettingsDomainEXT { @@ -1513,31 +1590,35 @@ typedef struct XrEventDataPerfSettingsEXT { typedef XrResult (XRAPI_PTR *PFN_xrPerfSettingsSetPerformanceLevelEXT)(XrSession session, XrPerfSettingsDomainEXT domain, XrPerfSettingsLevelEXT level); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrPerfSettingsSetPerformanceLevelEXT( XrSession session, XrPerfSettingsDomainEXT domain, XrPerfSettingsLevelEXT level); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_EXT_thermal_query 1 -#define XR_EXT_thermal_query_SPEC_VERSION 1 +#define XR_EXT_thermal_query_SPEC_VERSION 2 #define XR_EXT_THERMAL_QUERY_EXTENSION_NAME "XR_EXT_thermal_query" typedef XrResult (XRAPI_PTR *PFN_xrThermalGetTemperatureTrendEXT)(XrSession session, XrPerfSettingsDomainEXT domain, XrPerfSettingsNotificationLevelEXT* notificationLevel, float* tempHeadroom, float* tempSlope); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrThermalGetTemperatureTrendEXT( XrSession session, XrPerfSettingsDomainEXT domain, XrPerfSettingsNotificationLevelEXT* notificationLevel, float* tempHeadroom, float* tempSlope); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_EXT_debug_utils 1 XR_DEFINE_HANDLE(XrDebugUtilsMessengerEXT) -#define XR_EXT_debug_utils_SPEC_VERSION 3 +#define XR_EXT_debug_utils_SPEC_VERSION 4 #define XR_EXT_DEBUG_UTILS_EXTENSION_NAME "XR_EXT_debug_utils" typedef XrFlags64 XrDebugUtilsMessageSeverityFlagsEXT; @@ -1588,6 +1669,7 @@ typedef XrBool32 (XRAPI_PTR *PFN_xrDebugUtilsMessengerCallbackEXT)( void* userData); +// XrDebugUtilsMessengerCreateInfoEXT extends XrInstanceCreateInfo typedef struct XrDebugUtilsMessengerCreateInfoEXT { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -1606,6 +1688,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrSessionEndDebugUtilsLabelRegionEXT)(XrSession typedef XrResult (XRAPI_PTR *PFN_xrSessionInsertDebugUtilsLabelEXT)(XrSession session, const XrDebugUtilsLabelEXT* labelInfo); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrSetDebugUtilsObjectNameEXT( XrInstance instance, const XrDebugUtilsObjectNameInfoEXT* nameInfo); @@ -1634,18 +1717,21 @@ XRAPI_ATTR XrResult XRAPI_CALL xrSessionEndDebugUtilsLabelRegionEXT( XRAPI_ATTR XrResult XRAPI_CALL xrSessionInsertDebugUtilsLabelEXT( XrSession session, const XrDebugUtilsLabelEXT* labelInfo); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_EXT_eye_gaze_interaction 1 #define XR_EXT_eye_gaze_interaction_SPEC_VERSION 1 #define XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME "XR_EXT_eye_gaze_interaction" +// XrSystemEyeGazeInteractionPropertiesEXT extends XrSystemProperties typedef struct XrSystemEyeGazeInteractionPropertiesEXT { XrStructureType type; void* XR_MAY_ALIAS next; XrBool32 supportsEyeGazeInteraction; } XrSystemEyeGazeInteractionPropertiesEXT; +// XrEyeGazeSampleTimeEXT extends XrSpaceLocation typedef struct XrEyeGazeSampleTimeEXT { XrStructureType type; void* XR_MAY_ALIAS next; @@ -1655,18 +1741,18 @@ typedef struct XrEyeGazeSampleTimeEXT { #define XR_EXTX_overlay 1 -#define XR_EXTX_overlay_SPEC_VERSION 4 +#define XR_EXTX_overlay_SPEC_VERSION 5 #define XR_EXTX_OVERLAY_EXTENSION_NAME "XR_EXTX_overlay" typedef XrFlags64 XrOverlaySessionCreateFlagsEXTX; // Flag bits for XrOverlaySessionCreateFlagsEXTX -static const XrOverlaySessionCreateFlagsEXTX XR_OVERLAY_SESSION_CREATE_RELAXED_DISPLAY_TIME_BIT_EXTX = 0x00000001; typedef XrFlags64 XrOverlayMainSessionFlagsEXTX; // Flag bits for XrOverlayMainSessionFlagsEXTX static const XrOverlayMainSessionFlagsEXTX XR_OVERLAY_MAIN_SESSION_ENABLED_COMPOSITION_LAYER_INFO_DEPTH_BIT_EXTX = 0x00000001; +// XrSessionCreateInfoOverlayEXTX extends XrSessionCreateInfo typedef struct XrSessionCreateInfoOverlayEXTX { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -1695,7 +1781,7 @@ typedef struct XrEventDataMainSessionVisibilityChangedEXTX { #define XR_MSFT_spatial_anchor 1 XR_DEFINE_HANDLE(XrSpatialAnchorMSFT) -#define XR_MSFT_spatial_anchor_SPEC_VERSION 1 +#define XR_MSFT_spatial_anchor_SPEC_VERSION 2 #define XR_MSFT_SPATIAL_ANCHOR_EXTENSION_NAME "XR_MSFT_spatial_anchor" typedef struct XrSpatialAnchorCreateInfoMSFT { XrStructureType type; @@ -1717,6 +1803,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrCreateSpatialAnchorSpaceMSFT)(XrSession sessi typedef XrResult (XRAPI_PTR *PFN_xrDestroySpatialAnchorMSFT)(XrSpatialAnchorMSFT anchor); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrCreateSpatialAnchorMSFT( XrSession session, const XrSpatialAnchorCreateInfoMSFT* createInfo, @@ -1729,7 +1816,50 @@ XRAPI_ATTR XrResult XRAPI_CALL xrCreateSpatialAnchorSpaceMSFT( XRAPI_ATTR XrResult XRAPI_CALL xrDestroySpatialAnchorMSFT( XrSpatialAnchorMSFT anchor); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + + +#define XR_FB_composition_layer_image_layout 1 +#define XR_FB_composition_layer_image_layout_SPEC_VERSION 1 +#define XR_FB_COMPOSITION_LAYER_IMAGE_LAYOUT_EXTENSION_NAME "XR_FB_composition_layer_image_layout" +typedef XrFlags64 XrCompositionLayerImageLayoutFlagsFB; + +// Flag bits for XrCompositionLayerImageLayoutFlagsFB +static const XrCompositionLayerImageLayoutFlagsFB XR_COMPOSITION_LAYER_IMAGE_LAYOUT_VERTICAL_FLIP_BIT_FB = 0x00000001; + +// XrCompositionLayerImageLayoutFB extends XrCompositionLayerBaseHeader +typedef struct XrCompositionLayerImageLayoutFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + XrCompositionLayerImageLayoutFlagsFB flags; +} XrCompositionLayerImageLayoutFB; + + + +#define XR_FB_composition_layer_alpha_blend 1 +#define XR_FB_composition_layer_alpha_blend_SPEC_VERSION 2 +#define XR_FB_COMPOSITION_LAYER_ALPHA_BLEND_EXTENSION_NAME "XR_FB_composition_layer_alpha_blend" + +typedef enum XrBlendFactorFB { + XR_BLEND_FACTOR_ZERO_FB = 0, + XR_BLEND_FACTOR_ONE_FB = 1, + XR_BLEND_FACTOR_SRC_ALPHA_FB = 2, + XR_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA_FB = 3, + XR_BLEND_FACTOR_DST_ALPHA_FB = 4, + XR_BLEND_FACTOR_ONE_MINUS_DST_ALPHA_FB = 5, + XR_BLEND_FACTOR_MAX_ENUM_FB = 0x7FFFFFFF +} XrBlendFactorFB; +// XrCompositionLayerAlphaBlendFB extends XrCompositionLayerBaseHeader +typedef struct XrCompositionLayerAlphaBlendFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + XrBlendFactorFB srcFactorColor; + XrBlendFactorFB dstFactorColor; + XrBlendFactorFB srcFactorAlpha; + XrBlendFactorFB dstFactorAlpha; +} XrCompositionLayerAlphaBlendFB; + #define XR_MND_headless 1 @@ -1745,6 +1875,7 @@ XRAPI_ATTR XrResult XRAPI_CALL xrDestroySpatialAnchorMSFT( #define XR_EXT_view_configuration_depth_range 1 #define XR_EXT_view_configuration_depth_range_SPEC_VERSION 1 #define XR_EXT_VIEW_CONFIGURATION_DEPTH_RANGE_EXTENSION_NAME "XR_EXT_view_configuration_depth_range" +// XrViewConfigurationDepthRangeEXT extends XrViewConfigurationView typedef struct XrViewConfigurationDepthRangeEXT { XrStructureType type; void* XR_MAY_ALIAS next; @@ -1757,7 +1888,7 @@ typedef struct XrViewConfigurationDepthRangeEXT { #define XR_EXT_conformance_automation 1 -#define XR_EXT_conformance_automation_SPEC_VERSION 1 +#define XR_EXT_conformance_automation_SPEC_VERSION 3 #define XR_EXT_CONFORMANCE_AUTOMATION_EXTENSION_NAME "XR_EXT_conformance_automation" typedef XrResult (XRAPI_PTR *PFN_xrSetInputDeviceActiveEXT)(XrSession session, XrPath interactionProfile, XrPath topLevelPath, XrBool32 isActive); typedef XrResult (XRAPI_PTR *PFN_xrSetInputDeviceStateBoolEXT)(XrSession session, XrPath topLevelPath, XrPath inputSourcePath, XrBool32 state); @@ -1766,6 +1897,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrSetInputDeviceStateVector2fEXT)(XrSession ses typedef XrResult (XRAPI_PTR *PFN_xrSetInputDeviceLocationEXT)(XrSession session, XrPath topLevelPath, XrPath inputSourcePath, XrSpace space, XrPosef pose); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrSetInputDeviceActiveEXT( XrSession session, XrPath interactionProfile, @@ -1796,7 +1928,8 @@ XRAPI_ATTR XrResult XRAPI_CALL xrSetInputDeviceLocationEXT( XrPath inputSourcePath, XrSpace space, XrPosef pose); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_MSFT_spatial_graph_bridge 1 @@ -1819,11 +1952,13 @@ typedef struct XrSpatialGraphNodeSpaceCreateInfoMSFT { typedef XrResult (XRAPI_PTR *PFN_xrCreateSpatialGraphNodeSpaceMSFT)(XrSession session, const XrSpatialGraphNodeSpaceCreateInfoMSFT* createInfo, XrSpace* space); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrCreateSpatialGraphNodeSpaceMSFT( XrSession session, const XrSpatialGraphNodeSpaceCreateInfoMSFT* createInfo, XrSpace* space); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_MSFT_hand_interaction 1 @@ -1836,7 +1971,7 @@ XRAPI_ATTR XrResult XRAPI_CALL xrCreateSpatialGraphNodeSpaceMSFT( #define XR_HAND_JOINT_COUNT_EXT 26 XR_DEFINE_HANDLE(XrHandTrackerEXT) -#define XR_EXT_hand_tracking_SPEC_VERSION 2 +#define XR_EXT_hand_tracking_SPEC_VERSION 4 #define XR_EXT_HAND_TRACKING_EXTENSION_NAME "XR_EXT_hand_tracking" typedef enum XrHandEXT { @@ -1879,6 +2014,7 @@ typedef enum XrHandJointSetEXT { XR_HAND_JOINT_SET_DEFAULT_EXT = 0, XR_HAND_JOINT_SET_MAX_ENUM_EXT = 0x7FFFFFFF } XrHandJointSetEXT; +// XrSystemHandTrackingPropertiesEXT extends XrSystemProperties typedef struct XrSystemHandTrackingPropertiesEXT { XrStructureType type; void* XR_MAY_ALIAS next; @@ -1919,6 +2055,7 @@ typedef struct XrHandJointLocationsEXT { XrHandJointLocationEXT* jointLocations; } XrHandJointLocationsEXT; +// XrHandJointVelocitiesEXT extends XrHandJointLocationsEXT typedef struct XrHandJointVelocitiesEXT { XrStructureType type; void* XR_MAY_ALIAS next; @@ -1931,6 +2068,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrDestroyHandTrackerEXT)(XrHandTrackerEXT handT typedef XrResult (XRAPI_PTR *PFN_xrLocateHandJointsEXT)(XrHandTrackerEXT handTracker, const XrHandJointsLocateInfoEXT* locateInfo, XrHandJointLocationsEXT* locations); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrCreateHandTrackerEXT( XrSession session, const XrHandTrackerCreateInfoEXT* createInfo, @@ -1943,11 +2081,12 @@ XRAPI_ATTR XrResult XRAPI_CALL xrLocateHandJointsEXT( XrHandTrackerEXT handTracker, const XrHandJointsLocateInfoEXT* locateInfo, XrHandJointLocationsEXT* locations); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_MSFT_hand_tracking_mesh 1 -#define XR_MSFT_hand_tracking_mesh_SPEC_VERSION 2 +#define XR_MSFT_hand_tracking_mesh_SPEC_VERSION 3 #define XR_MSFT_HAND_TRACKING_MESH_EXTENSION_NAME "XR_MSFT_hand_tracking_mesh" typedef enum XrHandPoseTypeMSFT { @@ -1955,6 +2094,7 @@ typedef enum XrHandPoseTypeMSFT { XR_HAND_POSE_TYPE_REFERENCE_OPEN_PALM_MSFT = 1, XR_HAND_POSE_TYPE_MAX_ENUM_MSFT = 0x7FFFFFFF } XrHandPoseTypeMSFT; +// XrSystemHandTrackingMeshPropertiesMSFT extends XrSystemProperties typedef struct XrSystemHandTrackingMeshPropertiesMSFT { XrStructureType type; void* XR_MAY_ALIAS next; @@ -2006,6 +2146,7 @@ typedef struct XrHandMeshMSFT { XrHandMeshVertexBufferMSFT vertexBuffer; } XrHandMeshMSFT; +// XrHandPoseTypeInfoMSFT extends XrHandTrackerCreateInfoEXT typedef struct XrHandPoseTypeInfoMSFT { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -2016,6 +2157,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrCreateHandMeshSpaceMSFT)(XrHandTrackerEXT han typedef XrResult (XRAPI_PTR *PFN_xrUpdateHandMeshMSFT)(XrHandTrackerEXT handTracker, const XrHandMeshUpdateInfoMSFT* updateInfo, XrHandMeshMSFT* handMesh); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrCreateHandMeshSpaceMSFT( XrHandTrackerEXT handTracker, const XrHandMeshSpaceCreateInfoMSFT* createInfo, @@ -2025,12 +2167,14 @@ XRAPI_ATTR XrResult XRAPI_CALL xrUpdateHandMeshMSFT( XrHandTrackerEXT handTracker, const XrHandMeshUpdateInfoMSFT* updateInfo, XrHandMeshMSFT* handMesh); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_MSFT_secondary_view_configuration 1 #define XR_MSFT_secondary_view_configuration_SPEC_VERSION 1 #define XR_MSFT_SECONDARY_VIEW_CONFIGURATION_EXTENSION_NAME "XR_MSFT_secondary_view_configuration" +// XrSecondaryViewConfigurationSessionBeginInfoMSFT extends XrSessionBeginInfo typedef struct XrSecondaryViewConfigurationSessionBeginInfoMSFT { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -2045,6 +2189,7 @@ typedef struct XrSecondaryViewConfigurationStateMSFT { XrBool32 active; } XrSecondaryViewConfigurationStateMSFT; +// XrSecondaryViewConfigurationFrameStateMSFT extends XrFrameState typedef struct XrSecondaryViewConfigurationFrameStateMSFT { XrStructureType type; void* XR_MAY_ALIAS next; @@ -2061,6 +2206,7 @@ typedef struct XrSecondaryViewConfigurationLayerInfoMSFT { const XrCompositionLayerBaseHeader* const* layers; } XrSecondaryViewConfigurationLayerInfoMSFT; +// XrSecondaryViewConfigurationFrameEndInfoMSFT extends XrFrameEndInfo typedef struct XrSecondaryViewConfigurationFrameEndInfoMSFT { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -2068,6 +2214,7 @@ typedef struct XrSecondaryViewConfigurationFrameEndInfoMSFT { const XrSecondaryViewConfigurationLayerInfoMSFT* viewConfigurationLayersInfo; } XrSecondaryViewConfigurationFrameEndInfoMSFT; +// XrSecondaryViewConfigurationSwapchainCreateInfoMSFT extends XrSwapchainCreateInfo typedef struct XrSecondaryViewConfigurationSwapchainCreateInfoMSFT { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -2130,6 +2277,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrGetControllerModelPropertiesMSFT)(XrSession s typedef XrResult (XRAPI_PTR *PFN_xrGetControllerModelStateMSFT)(XrSession session, XrControllerModelKeyMSFT modelKey, XrControllerModelStateMSFT* state); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrGetControllerModelKeyMSFT( XrSession session, XrPath topLevelUserPath, @@ -2151,7 +2299,8 @@ XRAPI_ATTR XrResult XRAPI_CALL xrGetControllerModelStateMSFT( XrSession session, XrControllerModelKeyMSFT modelKey, XrControllerModelStateMSFT* state); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_EXT_win32_appcontainer_compatible 1 @@ -2162,6 +2311,7 @@ XRAPI_ATTR XrResult XRAPI_CALL xrGetControllerModelStateMSFT( #define XR_EPIC_view_configuration_fov 1 #define XR_EPIC_view_configuration_fov_SPEC_VERSION 2 #define XR_EPIC_VIEW_CONFIGURATION_FOV_EXTENSION_NAME "XR_EPIC_view_configuration_fov" +// XrViewConfigurationViewFovEPIC extends XrViewConfigurationView typedef struct XrViewConfigurationViewFovEPIC { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -2171,13 +2321,97 @@ typedef struct XrViewConfigurationViewFovEPIC { +#define XR_MSFT_composition_layer_reprojection 1 +#define XR_MSFT_composition_layer_reprojection_SPEC_VERSION 1 +#define XR_MSFT_COMPOSITION_LAYER_REPROJECTION_EXTENSION_NAME "XR_MSFT_composition_layer_reprojection" + +typedef enum XrReprojectionModeMSFT { + XR_REPROJECTION_MODE_DEPTH_MSFT = 1, + XR_REPROJECTION_MODE_PLANAR_FROM_DEPTH_MSFT = 2, + XR_REPROJECTION_MODE_PLANAR_MANUAL_MSFT = 3, + XR_REPROJECTION_MODE_ORIENTATION_ONLY_MSFT = 4, + XR_REPROJECTION_MODE_MAX_ENUM_MSFT = 0x7FFFFFFF +} XrReprojectionModeMSFT; +// XrCompositionLayerReprojectionInfoMSFT extends XrCompositionLayerProjection +typedef struct XrCompositionLayerReprojectionInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrReprojectionModeMSFT reprojectionMode; +} XrCompositionLayerReprojectionInfoMSFT; + +// XrCompositionLayerReprojectionPlaneOverrideMSFT extends XrCompositionLayerProjection +typedef struct XrCompositionLayerReprojectionPlaneOverrideMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrVector3f position; + XrVector3f normal; + XrVector3f velocity; +} XrCompositionLayerReprojectionPlaneOverrideMSFT; + +typedef XrResult (XRAPI_PTR *PFN_xrEnumerateReprojectionModesMSFT)(XrInstance instance, XrSystemId systemId, XrViewConfigurationType viewConfigurationType, uint32_t modeCapacityInput, uint32_t* modeCountOutput, XrReprojectionModeMSFT* modes); + +#ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES +XRAPI_ATTR XrResult XRAPI_CALL xrEnumerateReprojectionModesMSFT( + XrInstance instance, + XrSystemId systemId, + XrViewConfigurationType viewConfigurationType, + uint32_t modeCapacityInput, + uint32_t* modeCountOutput, + XrReprojectionModeMSFT* modes); +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + + #define XR_HUAWEI_controller_interaction 1 #define XR_HUAWEI_controller_interaction_SPEC_VERSION 1 #define XR_HUAWEI_CONTROLLER_INTERACTION_EXTENSION_NAME "XR_HUAWEI_controller_interaction" +#define XR_FB_swapchain_update_state 1 +#define XR_FB_swapchain_update_state_SPEC_VERSION 3 +#define XR_FB_SWAPCHAIN_UPDATE_STATE_EXTENSION_NAME "XR_FB_swapchain_update_state" +typedef struct XR_MAY_ALIAS XrSwapchainStateBaseHeaderFB { + XrStructureType type; + void* XR_MAY_ALIAS next; +} XrSwapchainStateBaseHeaderFB; + +typedef XrResult (XRAPI_PTR *PFN_xrUpdateSwapchainFB)(XrSwapchain swapchain, const XrSwapchainStateBaseHeaderFB* state); +typedef XrResult (XRAPI_PTR *PFN_xrGetSwapchainStateFB)(XrSwapchain swapchain, XrSwapchainStateBaseHeaderFB* state); + +#ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES +XRAPI_ATTR XrResult XRAPI_CALL xrUpdateSwapchainFB( + XrSwapchain swapchain, + const XrSwapchainStateBaseHeaderFB* state); + +XRAPI_ATTR XrResult XRAPI_CALL xrGetSwapchainStateFB( + XrSwapchain swapchain, + XrSwapchainStateBaseHeaderFB* state); +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + + +#define XR_FB_composition_layer_secure_content 1 +#define XR_FB_composition_layer_secure_content_SPEC_VERSION 1 +#define XR_FB_COMPOSITION_LAYER_SECURE_CONTENT_EXTENSION_NAME "XR_FB_composition_layer_secure_content" +typedef XrFlags64 XrCompositionLayerSecureContentFlagsFB; + +// Flag bits for XrCompositionLayerSecureContentFlagsFB +static const XrCompositionLayerSecureContentFlagsFB XR_COMPOSITION_LAYER_SECURE_CONTENT_EXCLUDE_LAYER_BIT_FB = 0x00000001; +static const XrCompositionLayerSecureContentFlagsFB XR_COMPOSITION_LAYER_SECURE_CONTENT_REPLACE_LAYER_BIT_FB = 0x00000002; + +// XrCompositionLayerSecureContentFB extends XrCompositionLayerBaseHeader +typedef struct XrCompositionLayerSecureContentFB { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrCompositionLayerSecureContentFlagsFB flags; +} XrCompositionLayerSecureContentFB; + + + #define XR_VALVE_analog_threshold 1 -#define XR_VALVE_analog_threshold_SPEC_VERSION 1 +#define XR_VALVE_analog_threshold_SPEC_VERSION 2 #define XR_VALVE_ANALOG_THRESHOLD_EXTENSION_NAME "XR_VALVE_analog_threshold" typedef struct XrInteractionProfileAnalogThresholdVALVE { XrStructureType type; @@ -2192,6 +2426,24 @@ typedef struct XrInteractionProfileAnalogThresholdVALVE { +#define XR_EXT_hand_joints_motion_range 1 +#define XR_EXT_hand_joints_motion_range_SPEC_VERSION 1 +#define XR_EXT_HAND_JOINTS_MOTION_RANGE_EXTENSION_NAME "XR_EXT_hand_joints_motion_range" + +typedef enum XrHandJointsMotionRangeEXT { + XR_HAND_JOINTS_MOTION_RANGE_UNOBSTRUCTED_EXT = 1, + XR_HAND_JOINTS_MOTION_RANGE_CONFORMING_TO_CONTROLLER_EXT = 2, + XR_HAND_JOINTS_MOTION_RANGE_MAX_ENUM_EXT = 0x7FFFFFFF +} XrHandJointsMotionRangeEXT; +// XrHandJointsMotionRangeInfoEXT extends XrHandJointsLocateInfoEXT +typedef struct XrHandJointsMotionRangeInfoEXT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrHandJointsMotionRangeEXT handJointsMotionRange; +} XrHandJointsMotionRangeInfoEXT; + + + #define XR_EXT_samsung_odyssey_controller 1 #define XR_EXT_samsung_odyssey_controller_SPEC_VERSION 1 #define XR_EXT_SAMSUNG_ODYSSEY_CONTROLLER_EXTENSION_NAME "XR_EXT_samsung_odyssey_controller" @@ -2207,6 +2459,374 @@ typedef struct XrInteractionProfileAnalogThresholdVALVE { #define XR_MND_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_EXTENSION_NAME "XR_MND_swapchain_usage_input_attachment_bit" +#define XR_MSFT_scene_understanding 1 + + XR_DEFINE_HANDLE(XrSceneObserverMSFT) + + + XR_DEFINE_HANDLE(XrSceneMSFT) + +#define XR_MSFT_scene_understanding_SPEC_VERSION 1 +#define XR_MSFT_SCENE_UNDERSTANDING_EXTENSION_NAME "XR_MSFT_scene_understanding" + +typedef enum XrSceneComputeFeatureMSFT { + XR_SCENE_COMPUTE_FEATURE_PLANE_MSFT = 1, + XR_SCENE_COMPUTE_FEATURE_PLANE_MESH_MSFT = 2, + XR_SCENE_COMPUTE_FEATURE_VISUAL_MESH_MSFT = 3, + XR_SCENE_COMPUTE_FEATURE_COLLIDER_MESH_MSFT = 4, + XR_SCENE_COMPUTE_FEATURE_SERIALIZE_SCENE_MSFT = 1000098000, + XR_SCENE_COMPUTE_FEATURE_MAX_ENUM_MSFT = 0x7FFFFFFF +} XrSceneComputeFeatureMSFT; + +typedef enum XrSceneComputeConsistencyMSFT { + XR_SCENE_COMPUTE_CONSISTENCY_SNAPSHOT_COMPLETE_MSFT = 1, + XR_SCENE_COMPUTE_CONSISTENCY_SNAPSHOT_INCOMPLETE_FAST_MSFT = 2, + XR_SCENE_COMPUTE_CONSISTENCY_OCCLUSION_OPTIMIZED_MSFT = 3, + XR_SCENE_COMPUTE_CONSISTENCY_MAX_ENUM_MSFT = 0x7FFFFFFF +} XrSceneComputeConsistencyMSFT; + +typedef enum XrMeshComputeLodMSFT { + XR_MESH_COMPUTE_LOD_COARSE_MSFT = 1, + XR_MESH_COMPUTE_LOD_MEDIUM_MSFT = 2, + XR_MESH_COMPUTE_LOD_FINE_MSFT = 3, + XR_MESH_COMPUTE_LOD_UNLIMITED_MSFT = 4, + XR_MESH_COMPUTE_LOD_MAX_ENUM_MSFT = 0x7FFFFFFF +} XrMeshComputeLodMSFT; + +typedef enum XrSceneComponentTypeMSFT { + XR_SCENE_COMPONENT_TYPE_INVALID_MSFT = -1, + XR_SCENE_COMPONENT_TYPE_OBJECT_MSFT = 1, + XR_SCENE_COMPONENT_TYPE_PLANE_MSFT = 2, + XR_SCENE_COMPONENT_TYPE_VISUAL_MESH_MSFT = 3, + XR_SCENE_COMPONENT_TYPE_COLLIDER_MESH_MSFT = 4, + XR_SCENE_COMPONENT_TYPE_SERIALIZED_SCENE_FRAGMENT_MSFT = 1000098000, + XR_SCENE_COMPONENT_TYPE_MAX_ENUM_MSFT = 0x7FFFFFFF +} XrSceneComponentTypeMSFT; + +typedef enum XrSceneObjectTypeMSFT { + XR_SCENE_OBJECT_TYPE_UNCATEGORIZED_MSFT = -1, + XR_SCENE_OBJECT_TYPE_BACKGROUND_MSFT = 1, + XR_SCENE_OBJECT_TYPE_WALL_MSFT = 2, + XR_SCENE_OBJECT_TYPE_FLOOR_MSFT = 3, + XR_SCENE_OBJECT_TYPE_CEILING_MSFT = 4, + XR_SCENE_OBJECT_TYPE_PLATFORM_MSFT = 5, + XR_SCENE_OBJECT_TYPE_INFERRED_MSFT = 6, + XR_SCENE_OBJECT_TYPE_MAX_ENUM_MSFT = 0x7FFFFFFF +} XrSceneObjectTypeMSFT; + +typedef enum XrScenePlaneAlignmentTypeMSFT { + XR_SCENE_PLANE_ALIGNMENT_TYPE_NON_ORTHOGONAL_MSFT = 0, + XR_SCENE_PLANE_ALIGNMENT_TYPE_HORIZONTAL_MSFT = 1, + XR_SCENE_PLANE_ALIGNMENT_TYPE_VERTICAL_MSFT = 2, + XR_SCENE_PLANE_ALIGNMENT_TYPE_MAX_ENUM_MSFT = 0x7FFFFFFF +} XrScenePlaneAlignmentTypeMSFT; + +typedef enum XrSceneComputeStateMSFT { + XR_SCENE_COMPUTE_STATE_NONE_MSFT = 0, + XR_SCENE_COMPUTE_STATE_UPDATING_MSFT = 1, + XR_SCENE_COMPUTE_STATE_COMPLETED_MSFT = 2, + XR_SCENE_COMPUTE_STATE_COMPLETED_WITH_ERROR_MSFT = 3, + XR_SCENE_COMPUTE_STATE_MAX_ENUM_MSFT = 0x7FFFFFFF +} XrSceneComputeStateMSFT; +typedef struct XrUuidMSFT { + uint8_t bytes[16]; +} XrUuidMSFT; + +typedef struct XrSceneObserverCreateInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; +} XrSceneObserverCreateInfoMSFT; + +typedef struct XrSceneCreateInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; +} XrSceneCreateInfoMSFT; + +typedef struct XrSceneSphereBoundMSFT { + XrVector3f center; + float radius; +} XrSceneSphereBoundMSFT; + +typedef struct XrSceneOrientedBoxBoundMSFT { + XrPosef pose; + XrVector3f extents; +} XrSceneOrientedBoxBoundMSFT; + +typedef struct XrSceneFrustumBoundMSFT { + XrPosef pose; + XrFovf fov; + float farDistance; +} XrSceneFrustumBoundMSFT; + +typedef struct XrSceneBoundsMSFT { + XrSpace space; + XrTime time; + uint32_t sphereCount; + const XrSceneSphereBoundMSFT* spheres; + uint32_t boxCount; + const XrSceneOrientedBoxBoundMSFT* boxes; + uint32_t frustumCount; + const XrSceneFrustumBoundMSFT* frustums; +} XrSceneBoundsMSFT; + +typedef struct XrNewSceneComputeInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + uint32_t requestedFeatureCount; + const XrSceneComputeFeatureMSFT* requestedFeatures; + XrSceneComputeConsistencyMSFT consistency; + XrSceneBoundsMSFT bounds; +} XrNewSceneComputeInfoMSFT; + +// XrVisualMeshComputeLodInfoMSFT extends XrNewSceneComputeInfoMSFT +typedef struct XrVisualMeshComputeLodInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrMeshComputeLodMSFT lod; +} XrVisualMeshComputeLodInfoMSFT; + +typedef struct XrSceneComponentMSFT { + XrSceneComponentTypeMSFT componentType; + XrUuidMSFT id; + XrUuidMSFT parentId; + XrTime updateTime; +} XrSceneComponentMSFT; + +typedef struct XrSceneComponentsMSFT { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t componentCapacityInput; + uint32_t componentCountOutput; + XrSceneComponentMSFT* components; +} XrSceneComponentsMSFT; + +typedef struct XrSceneComponentsGetInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrSceneComponentTypeMSFT componentType; +} XrSceneComponentsGetInfoMSFT; + +typedef struct XrSceneComponentLocationMSFT { + XrSpaceLocationFlags flags; + XrPosef pose; +} XrSceneComponentLocationMSFT; + +typedef struct XrSceneComponentLocationsMSFT { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t locationCount; + XrSceneComponentLocationMSFT* locations; +} XrSceneComponentLocationsMSFT; + +typedef struct XrSceneComponentsLocateInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrSpace baseSpace; + XrTime time; + uint32_t componentIdCount; + const XrUuidMSFT* componentIds; +} XrSceneComponentsLocateInfoMSFT; + +typedef struct XrSceneObjectMSFT { + XrSceneObjectTypeMSFT objectType; +} XrSceneObjectMSFT; + +// XrSceneObjectsMSFT extends XrSceneComponentsMSFT +typedef struct XrSceneObjectsMSFT { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t sceneObjectCount; + XrSceneObjectMSFT* sceneObjects; +} XrSceneObjectsMSFT; + +// XrSceneComponentParentFilterInfoMSFT extends XrSceneComponentsGetInfoMSFT +typedef struct XrSceneComponentParentFilterInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrUuidMSFT parentId; +} XrSceneComponentParentFilterInfoMSFT; + +// XrSceneObjectTypesFilterInfoMSFT extends XrSceneComponentsGetInfoMSFT +typedef struct XrSceneObjectTypesFilterInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + uint32_t objectTypeCount; + const XrSceneObjectTypeMSFT* objectTypes; +} XrSceneObjectTypesFilterInfoMSFT; + +typedef struct XrScenePlaneMSFT { + XrScenePlaneAlignmentTypeMSFT alignment; + XrExtent2Df size; + uint64_t meshBufferId; + XrBool32 supportsIndicesUint16; +} XrScenePlaneMSFT; + +// XrScenePlanesMSFT extends XrSceneComponentsMSFT +typedef struct XrScenePlanesMSFT { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t scenePlaneCount; + XrScenePlaneMSFT* scenePlanes; +} XrScenePlanesMSFT; + +// XrScenePlaneAlignmentFilterInfoMSFT extends XrSceneComponentsGetInfoMSFT +typedef struct XrScenePlaneAlignmentFilterInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + uint32_t alignmentCount; + const XrScenePlaneAlignmentTypeMSFT* alignments; +} XrScenePlaneAlignmentFilterInfoMSFT; + +typedef struct XrSceneMeshMSFT { + uint64_t meshBufferId; + XrBool32 supportsIndicesUint16; +} XrSceneMeshMSFT; + +// XrSceneMeshesMSFT extends XrSceneComponentsMSFT +typedef struct XrSceneMeshesMSFT { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t sceneMeshCount; + XrSceneMeshMSFT* sceneMeshes; +} XrSceneMeshesMSFT; + +typedef struct XrSceneMeshBuffersGetInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + uint64_t meshBufferId; +} XrSceneMeshBuffersGetInfoMSFT; + +typedef struct XrSceneMeshBuffersMSFT { + XrStructureType type; + void* XR_MAY_ALIAS next; +} XrSceneMeshBuffersMSFT; + +typedef struct XrSceneMeshVertexBufferMSFT { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t vertexCapacityInput; + uint32_t vertexCountOutput; + XrVector3f* vertices; +} XrSceneMeshVertexBufferMSFT; + +typedef struct XrSceneMeshIndicesUint32MSFT { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t indexCapacityInput; + uint32_t indexCountOutput; + uint32_t* indices; +} XrSceneMeshIndicesUint32MSFT; + +typedef struct XrSceneMeshIndicesUint16MSFT { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t indexCapacityInput; + uint32_t indexCountOutput; + uint16_t* indices; +} XrSceneMeshIndicesUint16MSFT; + +typedef XrResult (XRAPI_PTR *PFN_xrEnumerateSceneComputeFeaturesMSFT)(XrInstance instance, XrSystemId systemId, uint32_t featureCapacityInput, uint32_t* featureCountOutput, XrSceneComputeFeatureMSFT* features); +typedef XrResult (XRAPI_PTR *PFN_xrCreateSceneObserverMSFT)(XrSession session, const XrSceneObserverCreateInfoMSFT* createInfo, XrSceneObserverMSFT* sceneObserver); +typedef XrResult (XRAPI_PTR *PFN_xrDestroySceneObserverMSFT)(XrSceneObserverMSFT sceneObserver); +typedef XrResult (XRAPI_PTR *PFN_xrCreateSceneMSFT)(XrSceneObserverMSFT sceneObserver, const XrSceneCreateInfoMSFT* createInfo, XrSceneMSFT* scene); +typedef XrResult (XRAPI_PTR *PFN_xrDestroySceneMSFT)(XrSceneMSFT scene); +typedef XrResult (XRAPI_PTR *PFN_xrComputeNewSceneMSFT)(XrSceneObserverMSFT sceneObserver, const XrNewSceneComputeInfoMSFT* computeInfo); +typedef XrResult (XRAPI_PTR *PFN_xrGetSceneComputeStateMSFT)(XrSceneObserverMSFT sceneObserver, XrSceneComputeStateMSFT* state); +typedef XrResult (XRAPI_PTR *PFN_xrGetSceneComponentsMSFT)(XrSceneMSFT scene, const XrSceneComponentsGetInfoMSFT* getInfo, XrSceneComponentsMSFT* components); +typedef XrResult (XRAPI_PTR *PFN_xrLocateSceneComponentsMSFT)(XrSceneMSFT scene, const XrSceneComponentsLocateInfoMSFT* locateInfo, XrSceneComponentLocationsMSFT* locations); +typedef XrResult (XRAPI_PTR *PFN_xrGetSceneMeshBuffersMSFT)(XrSceneMSFT scene, const XrSceneMeshBuffersGetInfoMSFT* getInfo, XrSceneMeshBuffersMSFT* buffers); + +#ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES +XRAPI_ATTR XrResult XRAPI_CALL xrEnumerateSceneComputeFeaturesMSFT( + XrInstance instance, + XrSystemId systemId, + uint32_t featureCapacityInput, + uint32_t* featureCountOutput, + XrSceneComputeFeatureMSFT* features); + +XRAPI_ATTR XrResult XRAPI_CALL xrCreateSceneObserverMSFT( + XrSession session, + const XrSceneObserverCreateInfoMSFT* createInfo, + XrSceneObserverMSFT* sceneObserver); + +XRAPI_ATTR XrResult XRAPI_CALL xrDestroySceneObserverMSFT( + XrSceneObserverMSFT sceneObserver); + +XRAPI_ATTR XrResult XRAPI_CALL xrCreateSceneMSFT( + XrSceneObserverMSFT sceneObserver, + const XrSceneCreateInfoMSFT* createInfo, + XrSceneMSFT* scene); + +XRAPI_ATTR XrResult XRAPI_CALL xrDestroySceneMSFT( + XrSceneMSFT scene); + +XRAPI_ATTR XrResult XRAPI_CALL xrComputeNewSceneMSFT( + XrSceneObserverMSFT sceneObserver, + const XrNewSceneComputeInfoMSFT* computeInfo); + +XRAPI_ATTR XrResult XRAPI_CALL xrGetSceneComputeStateMSFT( + XrSceneObserverMSFT sceneObserver, + XrSceneComputeStateMSFT* state); + +XRAPI_ATTR XrResult XRAPI_CALL xrGetSceneComponentsMSFT( + XrSceneMSFT scene, + const XrSceneComponentsGetInfoMSFT* getInfo, + XrSceneComponentsMSFT* components); + +XRAPI_ATTR XrResult XRAPI_CALL xrLocateSceneComponentsMSFT( + XrSceneMSFT scene, + const XrSceneComponentsLocateInfoMSFT* locateInfo, + XrSceneComponentLocationsMSFT* locations); + +XRAPI_ATTR XrResult XRAPI_CALL xrGetSceneMeshBuffersMSFT( + XrSceneMSFT scene, + const XrSceneMeshBuffersGetInfoMSFT* getInfo, + XrSceneMeshBuffersMSFT* buffers); +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + + +#define XR_MSFT_scene_understanding_serialization 1 +#define XR_MSFT_scene_understanding_serialization_SPEC_VERSION 1 +#define XR_MSFT_SCENE_UNDERSTANDING_SERIALIZATION_EXTENSION_NAME "XR_MSFT_scene_understanding_serialization" +typedef struct XrSerializedSceneFragmentDataGetInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrUuidMSFT sceneFragmentId; +} XrSerializedSceneFragmentDataGetInfoMSFT; + +typedef struct XrDeserializeSceneFragmentMSFT { + uint32_t bufferSize; + const uint8_t* buffer; +} XrDeserializeSceneFragmentMSFT; + +typedef struct XrSceneDeserializeInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + uint32_t fragmentCount; + const XrDeserializeSceneFragmentMSFT* fragments; +} XrSceneDeserializeInfoMSFT; + +typedef XrResult (XRAPI_PTR *PFN_xrDeserializeSceneMSFT)(XrSceneObserverMSFT sceneObserver, const XrSceneDeserializeInfoMSFT* deserializeInfo); +typedef XrResult (XRAPI_PTR *PFN_xrGetSerializedSceneFragmentDataMSFT)(XrSceneMSFT scene, const XrSerializedSceneFragmentDataGetInfoMSFT* getInfo, uint32_t countInput, uint32_t* readOutput, uint8_t* buffer); + +#ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES +XRAPI_ATTR XrResult XRAPI_CALL xrDeserializeSceneMSFT( + XrSceneObserverMSFT sceneObserver, + const XrSceneDeserializeInfoMSFT* deserializeInfo); + +XRAPI_ATTR XrResult XRAPI_CALL xrGetSerializedSceneFragmentDataMSFT( + XrSceneMSFT scene, + const XrSerializedSceneFragmentDataGetInfoMSFT* getInfo, + uint32_t countInput, + uint32_t* readOutput, + uint8_t* buffer); +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + + #define XR_FB_display_refresh_rate 1 #define XR_FB_display_refresh_rate_SPEC_VERSION 1 #define XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME "XR_FB_display_refresh_rate" @@ -2222,6 +2842,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrGetDisplayRefreshRateFB)(XrSession session, f typedef XrResult (XRAPI_PTR *PFN_xrRequestDisplayRefreshRateFB)(XrSession session, float displayRefreshRate); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrEnumerateDisplayRefreshRatesFB( XrSession session, uint32_t displayRefreshRateCapacityInput, @@ -2235,7 +2856,8 @@ XRAPI_ATTR XrResult XRAPI_CALL xrGetDisplayRefreshRateFB( XRAPI_ATTR XrResult XRAPI_CALL xrRequestDisplayRefreshRateFB( XrSession session, float displayRefreshRate); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #define XR_HTC_vive_cosmos_controller_interaction 1 @@ -2268,6 +2890,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrEnumerateColorSpacesFB)(XrSession session, ui typedef XrResult (XRAPI_PTR *PFN_xrSetColorSpaceFB)(XrSession session, const XrColorSpaceFB colorspace); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrEnumerateColorSpacesFB( XrSession session, uint32_t colorSpaceCapacityInput, @@ -2277,7 +2900,335 @@ XRAPI_ATTR XrResult XRAPI_CALL xrEnumerateColorSpacesFB( XRAPI_ATTR XrResult XRAPI_CALL xrSetColorSpaceFB( XrSession session, const XrColorSpaceFB colorspace); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + + +#define XR_FB_hand_tracking_mesh 1 +#define XR_FB_hand_tracking_mesh_SPEC_VERSION 1 +#define XR_FB_HAND_TRACKING_MESH_EXTENSION_NAME "XR_FB_hand_tracking_mesh" +typedef struct XrVector4sFB { + int16_t x; + int16_t y; + int16_t z; + int16_t w; +} XrVector4sFB; + +typedef struct XrHandTrackingMeshFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t jointCapacityInput; + uint32_t jointCountOutput; + XrPosef* jointBindPoses; + float* jointRadii; + XrHandJointEXT* jointParents; + uint32_t vertexCapacityInput; + uint32_t vertexCountOutput; + XrVector3f* vertexPositions; + XrVector3f* vertexNormals; + XrVector2f* vertexUVs; + XrVector4sFB* vertexBlendIndices; + XrVector4f* vertexBlendWeights; + uint32_t indexCapacityInput; + uint32_t indexCountOutput; + int16_t* indices; +} XrHandTrackingMeshFB; + +// XrHandTrackingScaleFB extends XrHandJointsLocateInfoEXT +typedef struct XrHandTrackingScaleFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + float sensorOutput; + float currentOutput; + XrBool32 overrideHandScale; + float overrideValueInput; +} XrHandTrackingScaleFB; + +typedef XrResult (XRAPI_PTR *PFN_xrGetHandMeshFB)(XrHandTrackerEXT handTracker, XrHandTrackingMeshFB* mesh); + +#ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES +XRAPI_ATTR XrResult XRAPI_CALL xrGetHandMeshFB( + XrHandTrackerEXT handTracker, + XrHandTrackingMeshFB* mesh); +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + + +#define XR_FB_hand_tracking_aim 1 +#define XR_FB_hand_tracking_aim_SPEC_VERSION 1 +#define XR_FB_HAND_TRACKING_AIM_EXTENSION_NAME "XR_FB_hand_tracking_aim" +typedef XrFlags64 XrHandTrackingAimFlagsFB; + +// Flag bits for XrHandTrackingAimFlagsFB +static const XrHandTrackingAimFlagsFB XR_HAND_TRACKING_AIM_COMPUTED_BIT_FB = 0x00000001; +static const XrHandTrackingAimFlagsFB XR_HAND_TRACKING_AIM_VALID_BIT_FB = 0x00000002; +static const XrHandTrackingAimFlagsFB XR_HAND_TRACKING_AIM_INDEX_PINCHING_BIT_FB = 0x00000004; +static const XrHandTrackingAimFlagsFB XR_HAND_TRACKING_AIM_MIDDLE_PINCHING_BIT_FB = 0x00000008; +static const XrHandTrackingAimFlagsFB XR_HAND_TRACKING_AIM_RING_PINCHING_BIT_FB = 0x00000010; +static const XrHandTrackingAimFlagsFB XR_HAND_TRACKING_AIM_LITTLE_PINCHING_BIT_FB = 0x00000020; +static const XrHandTrackingAimFlagsFB XR_HAND_TRACKING_AIM_SYSTEM_GESTURE_BIT_FB = 0x00000040; +static const XrHandTrackingAimFlagsFB XR_HAND_TRACKING_AIM_DOMINANT_HAND_BIT_FB = 0x00000080; +static const XrHandTrackingAimFlagsFB XR_HAND_TRACKING_AIM_MENU_PRESSED_BIT_FB = 0x00000100; + +// XrHandTrackingAimStateFB extends XrHandJointsLocateInfoEXT +typedef struct XrHandTrackingAimStateFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + XrHandTrackingAimFlagsFB status; + XrPosef aimPose; + float pinchStrengthIndex; + float pinchStrengthMiddle; + float pinchStrengthRing; + float pinchStrengthLittle; +} XrHandTrackingAimStateFB; + + + +#define XR_FB_hand_tracking_capsules 1 +#define XR_FB_HAND_TRACKING_CAPSULE_POINT_COUNT 2 +#define XR_FB_HAND_TRACKING_CAPSULE_COUNT 19 +#define XR_FB_hand_tracking_capsules_SPEC_VERSION 1 +#define XR_FB_HAND_TRACKING_CAPSULES_EXTENSION_NAME "XR_FB_hand_tracking_capsules" +typedef struct XrHandCapsuleFB { + XrVector3f points[XR_FB_HAND_TRACKING_CAPSULE_POINT_COUNT]; + float radius; + XrHandJointEXT joint; +} XrHandCapsuleFB; + +// XrHandTrackingCapsulesStateFB extends XrHandJointsLocateInfoEXT +typedef struct XrHandTrackingCapsulesStateFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + XrHandCapsuleFB capsules[XR_FB_HAND_TRACKING_CAPSULE_COUNT]; +} XrHandTrackingCapsulesStateFB; + + + +#define XR_FB_foveation 1 +XR_DEFINE_HANDLE(XrFoveationProfileFB) +#define XR_FB_foveation_SPEC_VERSION 1 +#define XR_FB_FOVEATION_EXTENSION_NAME "XR_FB_foveation" +typedef XrFlags64 XrSwapchainCreateFoveationFlagsFB; + +// Flag bits for XrSwapchainCreateFoveationFlagsFB +static const XrSwapchainCreateFoveationFlagsFB XR_SWAPCHAIN_CREATE_FOVEATION_SCALED_BIN_BIT_FB = 0x00000001; +static const XrSwapchainCreateFoveationFlagsFB XR_SWAPCHAIN_CREATE_FOVEATION_FRAGMENT_DENSITY_MAP_BIT_FB = 0x00000002; + +typedef XrFlags64 XrSwapchainStateFoveationFlagsFB; + +// Flag bits for XrSwapchainStateFoveationFlagsFB + +typedef struct XrFoveationProfileCreateInfoFB { + XrStructureType type; + void* XR_MAY_ALIAS next; +} XrFoveationProfileCreateInfoFB; + +// XrSwapchainCreateInfoFoveationFB extends XrSwapchainCreateInfo +typedef struct XrSwapchainCreateInfoFoveationFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + XrSwapchainCreateFoveationFlagsFB flags; +} XrSwapchainCreateInfoFoveationFB; + +typedef struct XrSwapchainStateFoveationFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + XrSwapchainStateFoveationFlagsFB flags; + XrFoveationProfileFB profile; +} XrSwapchainStateFoveationFB; + +typedef XrResult (XRAPI_PTR *PFN_xrCreateFoveationProfileFB)(XrSession session, const XrFoveationProfileCreateInfoFB* createInfo, XrFoveationProfileFB* profile); +typedef XrResult (XRAPI_PTR *PFN_xrDestroyFoveationProfileFB)(XrFoveationProfileFB profile); + +#ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES +XRAPI_ATTR XrResult XRAPI_CALL xrCreateFoveationProfileFB( + XrSession session, + const XrFoveationProfileCreateInfoFB* createInfo, + XrFoveationProfileFB* profile); + +XRAPI_ATTR XrResult XRAPI_CALL xrDestroyFoveationProfileFB( + XrFoveationProfileFB profile); +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + + +#define XR_FB_foveation_configuration 1 +#define XR_FB_foveation_configuration_SPEC_VERSION 1 +#define XR_FB_FOVEATION_CONFIGURATION_EXTENSION_NAME "XR_FB_foveation_configuration" + +typedef enum XrFoveationLevelFB { + XR_FOVEATION_LEVEL_NONE_FB = 0, + XR_FOVEATION_LEVEL_LOW_FB = 1, + XR_FOVEATION_LEVEL_MEDIUM_FB = 2, + XR_FOVEATION_LEVEL_HIGH_FB = 3, + XR_FOVEATION_LEVEL_MAX_ENUM_FB = 0x7FFFFFFF +} XrFoveationLevelFB; + +typedef enum XrFoveationDynamicFB { + XR_FOVEATION_DYNAMIC_DISABLED_FB = 0, + XR_FOVEATION_DYNAMIC_LEVEL_ENABLED_FB = 1, + XR_FOVEATION_DYNAMIC_MAX_ENUM_FB = 0x7FFFFFFF +} XrFoveationDynamicFB; +// XrFoveationLevelProfileCreateInfoFB extends XrFoveationProfileCreateInfoFB +typedef struct XrFoveationLevelProfileCreateInfoFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + XrFoveationLevelFB level; + float verticalOffset; + XrFoveationDynamicFB dynamic; +} XrFoveationLevelProfileCreateInfoFB; + + + +#define XR_VARJO_foveated_rendering 1 +#define XR_VARJO_foveated_rendering_SPEC_VERSION 2 +#define XR_VARJO_FOVEATED_RENDERING_EXTENSION_NAME "XR_VARJO_foveated_rendering" +// XrViewLocateFoveatedRenderingVARJO extends XrViewLocateInfo +typedef struct XrViewLocateFoveatedRenderingVARJO { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrBool32 foveatedRenderingActive; +} XrViewLocateFoveatedRenderingVARJO; + +// XrFoveatedViewConfigurationViewVARJO extends XrViewConfigurationView +typedef struct XrFoveatedViewConfigurationViewVARJO { + XrStructureType type; + void* XR_MAY_ALIAS next; + XrBool32 foveatedRenderingActive; +} XrFoveatedViewConfigurationViewVARJO; + +// XrSystemFoveatedRenderingPropertiesVARJO extends XrSystemProperties +typedef struct XrSystemFoveatedRenderingPropertiesVARJO { + XrStructureType type; + void* XR_MAY_ALIAS next; + XrBool32 supportsFoveatedRendering; +} XrSystemFoveatedRenderingPropertiesVARJO; + + + +#define XR_VARJO_composition_layer_depth_test 1 +#define XR_VARJO_composition_layer_depth_test_SPEC_VERSION 2 +#define XR_VARJO_COMPOSITION_LAYER_DEPTH_TEST_EXTENSION_NAME "XR_VARJO_composition_layer_depth_test" +// XrCompositionLayerDepthTestVARJO extends XrCompositionLayerProjection +typedef struct XrCompositionLayerDepthTestVARJO { + XrStructureType type; + const void* XR_MAY_ALIAS next; + float depthTestRangeNearZ; + float depthTestRangeFarZ; +} XrCompositionLayerDepthTestVARJO; + + + +#define XR_VARJO_environment_depth_estimation 1 +#define XR_VARJO_environment_depth_estimation_SPEC_VERSION 1 +#define XR_VARJO_ENVIRONMENT_DEPTH_ESTIMATION_EXTENSION_NAME "XR_VARJO_environment_depth_estimation" +typedef XrResult (XRAPI_PTR *PFN_xrSetEnvironmentDepthEstimationVARJO)(XrSession session, XrBool32 enabled); + +#ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES +XRAPI_ATTR XrResult XRAPI_CALL xrSetEnvironmentDepthEstimationVARJO( + XrSession session, + XrBool32 enabled); +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + + +#define XR_MSFT_spatial_anchor_persistence 1 +XR_DEFINE_HANDLE(XrSpatialAnchorStoreConnectionMSFT) +#define XR_MAX_SPATIAL_ANCHOR_NAME_SIZE_MSFT 256 +#define XR_MSFT_spatial_anchor_persistence_SPEC_VERSION 2 +#define XR_MSFT_SPATIAL_ANCHOR_PERSISTENCE_EXTENSION_NAME "XR_MSFT_spatial_anchor_persistence" +typedef struct XrSpatialAnchorPersistenceNameMSFT { + char name[XR_MAX_SPATIAL_ANCHOR_NAME_SIZE_MSFT]; +} XrSpatialAnchorPersistenceNameMSFT; + +typedef struct XrSpatialAnchorPersistenceInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrSpatialAnchorPersistenceNameMSFT spatialAnchorPersistenceName; + XrSpatialAnchorMSFT spatialAnchor; +} XrSpatialAnchorPersistenceInfoMSFT; + +typedef struct XrSpatialAnchorFromPersistedAnchorCreateInfoMSFT { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore; + XrSpatialAnchorPersistenceNameMSFT spatialAnchorPersistenceName; +} XrSpatialAnchorFromPersistedAnchorCreateInfoMSFT; + +typedef XrResult (XRAPI_PTR *PFN_xrCreateSpatialAnchorStoreConnectionMSFT)(XrSession session, XrSpatialAnchorStoreConnectionMSFT* spatialAnchorStore); +typedef XrResult (XRAPI_PTR *PFN_xrDestroySpatialAnchorStoreConnectionMSFT)(XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore); +typedef XrResult (XRAPI_PTR *PFN_xrPersistSpatialAnchorMSFT)(XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore, const XrSpatialAnchorPersistenceInfoMSFT* spatialAnchorPersistenceInfo); +typedef XrResult (XRAPI_PTR *PFN_xrEnumeratePersistedSpatialAnchorNamesMSFT)(XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore, uint32_t spatialAnchorNamesCapacityInput, uint32_t* spatialAnchorNamesCountOutput, XrSpatialAnchorPersistenceNameMSFT* persistedAnchorNames); +typedef XrResult (XRAPI_PTR *PFN_xrCreateSpatialAnchorFromPersistedNameMSFT)(XrSession session, const XrSpatialAnchorFromPersistedAnchorCreateInfoMSFT* spatialAnchorCreateInfo, XrSpatialAnchorMSFT* spatialAnchor); +typedef XrResult (XRAPI_PTR *PFN_xrUnpersistSpatialAnchorMSFT)(XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore, const XrSpatialAnchorPersistenceNameMSFT* spatialAnchorPersistenceName); +typedef XrResult (XRAPI_PTR *PFN_xrClearSpatialAnchorStoreMSFT)(XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore); + +#ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES +XRAPI_ATTR XrResult XRAPI_CALL xrCreateSpatialAnchorStoreConnectionMSFT( + XrSession session, + XrSpatialAnchorStoreConnectionMSFT* spatialAnchorStore); + +XRAPI_ATTR XrResult XRAPI_CALL xrDestroySpatialAnchorStoreConnectionMSFT( + XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore); + +XRAPI_ATTR XrResult XRAPI_CALL xrPersistSpatialAnchorMSFT( + XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore, + const XrSpatialAnchorPersistenceInfoMSFT* spatialAnchorPersistenceInfo); + +XRAPI_ATTR XrResult XRAPI_CALL xrEnumeratePersistedSpatialAnchorNamesMSFT( + XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore, + uint32_t spatialAnchorNamesCapacityInput, + uint32_t* spatialAnchorNamesCountOutput, + XrSpatialAnchorPersistenceNameMSFT* persistedAnchorNames); + +XRAPI_ATTR XrResult XRAPI_CALL xrCreateSpatialAnchorFromPersistedNameMSFT( + XrSession session, + const XrSpatialAnchorFromPersistedAnchorCreateInfoMSFT* spatialAnchorCreateInfo, + XrSpatialAnchorMSFT* spatialAnchor); + +XRAPI_ATTR XrResult XRAPI_CALL xrUnpersistSpatialAnchorMSFT( + XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore, + const XrSpatialAnchorPersistenceNameMSFT* spatialAnchorPersistenceName); + +XRAPI_ATTR XrResult XRAPI_CALL xrClearSpatialAnchorStoreMSFT( + XrSpatialAnchorStoreConnectionMSFT spatialAnchorStore); +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + + +#define XR_FB_space_warp 1 +#define XR_FB_space_warp_SPEC_VERSION 1 +#define XR_FB_SPACE_WARP_EXTENSION_NAME "XR_FB_space_warp" +typedef XrFlags64 XrCompositionLayerSpaceWarpInfoFlagsFB; + +// Flag bits for XrCompositionLayerSpaceWarpInfoFlagsFB + +// XrCompositionLayerSpaceWarpInfoFB extends XrCompositionLayerProjectionView +typedef struct XrCompositionLayerSpaceWarpInfoFB { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrCompositionLayerSpaceWarpInfoFlagsFB layerFlags; + XrSwapchainSubImage motionVectorSubImage; + XrPosef appSpaceDeltaPose; + XrSwapchainSubImage depthSubImage; + float minDepth; + float maxDepth; + float nearZ; + float farZ; +} XrCompositionLayerSpaceWarpInfoFB; + +// XrSystemSpaceWarpPropertiesFB extends XrSystemProperties +typedef struct XrSystemSpaceWarpPropertiesFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t recommendedMotionVectorImageRectWidth; + uint32_t recommendedMotionVectorImageRectHeight; +} XrSystemSpaceWarpPropertiesFB; + #ifdef __cplusplus } diff --git a/src/external/openxr_includes/openxr/openxr_platform.h b/src/external/openxr_includes/openxr/openxr_platform.h index e29598db5..8f8427f5b 100644 --- a/src/external/openxr_includes/openxr/openxr_platform.h +++ b/src/external/openxr_includes/openxr/openxr_platform.h @@ -2,7 +2,7 @@ #define OPENXR_PLATFORM_H_ 1 /* -** Copyright (c) 2017-2020 The Khronos Group Inc. +** Copyright (c) 2017-2021, The Khronos Group Inc. ** ** SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -35,11 +35,13 @@ typedef enum XrAndroidThreadTypeKHR { typedef XrResult (XRAPI_PTR *PFN_xrSetAndroidApplicationThreadKHR)(XrSession session, XrAndroidThreadTypeKHR threadType, uint32_t threadId); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrSetAndroidApplicationThreadKHR( XrSession session, XrAndroidThreadTypeKHR threadType, uint32_t threadId); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_PLATFORM_ANDROID */ #ifdef XR_USE_PLATFORM_ANDROID @@ -50,12 +52,14 @@ XRAPI_ATTR XrResult XRAPI_CALL xrSetAndroidApplicationThreadKHR( typedef XrResult (XRAPI_PTR *PFN_xrCreateSwapchainAndroidSurfaceKHR)(XrSession session, const XrSwapchainCreateInfo* info, XrSwapchain* swapchain, jobject* surface); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrCreateSwapchainAndroidSurfaceKHR( XrSession session, const XrSwapchainCreateInfo* info, XrSwapchain* swapchain, jobject* surface); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_PLATFORM_ANDROID */ #ifdef XR_USE_PLATFORM_ANDROID @@ -63,6 +67,7 @@ XRAPI_ATTR XrResult XRAPI_CALL xrCreateSwapchainAndroidSurfaceKHR( #define XR_KHR_android_create_instance 1 #define XR_KHR_android_create_instance_SPEC_VERSION 3 #define XR_KHR_ANDROID_CREATE_INSTANCE_EXTENSION_NAME "XR_KHR_android_create_instance" +// XrInstanceCreateInfoAndroidKHR extends XrInstanceCreateInfo typedef struct XrInstanceCreateInfoAndroidKHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -75,7 +80,7 @@ typedef struct XrInstanceCreateInfoAndroidKHR { #ifdef XR_USE_GRAPHICS_API_VULKAN #define XR_KHR_vulkan_swapchain_format_list 1 -#define XR_KHR_vulkan_swapchain_format_list_SPEC_VERSION 3 +#define XR_KHR_vulkan_swapchain_format_list_SPEC_VERSION 4 #define XR_KHR_VULKAN_SWAPCHAIN_FORMAT_LIST_EXTENSION_NAME "XR_KHR_vulkan_swapchain_format_list" typedef struct XrVulkanSwapchainFormatListCreateInfoKHR { XrStructureType type; @@ -92,6 +97,7 @@ typedef struct XrVulkanSwapchainFormatListCreateInfoKHR { #define XR_KHR_opengl_enable_SPEC_VERSION 9 #define XR_KHR_OPENGL_ENABLE_EXTENSION_NAME "XR_KHR_opengl_enable" #ifdef XR_USE_PLATFORM_WIN32 +// XrGraphicsBindingOpenGLWin32KHR extends XrSessionCreateInfo typedef struct XrGraphicsBindingOpenGLWin32KHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -101,6 +107,7 @@ typedef struct XrGraphicsBindingOpenGLWin32KHR { #endif // XR_USE_PLATFORM_WIN32 #ifdef XR_USE_PLATFORM_XLIB +// XrGraphicsBindingOpenGLXlibKHR extends XrSessionCreateInfo typedef struct XrGraphicsBindingOpenGLXlibKHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -113,6 +120,7 @@ typedef struct XrGraphicsBindingOpenGLXlibKHR { #endif // XR_USE_PLATFORM_XLIB #ifdef XR_USE_PLATFORM_XCB +// XrGraphicsBindingOpenGLXcbKHR extends XrSessionCreateInfo typedef struct XrGraphicsBindingOpenGLXcbKHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -126,6 +134,7 @@ typedef struct XrGraphicsBindingOpenGLXcbKHR { #endif // XR_USE_PLATFORM_XCB #ifdef XR_USE_PLATFORM_WAYLAND +// XrGraphicsBindingOpenGLWaylandKHR extends XrSessionCreateInfo typedef struct XrGraphicsBindingOpenGLWaylandKHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -149,11 +158,13 @@ typedef struct XrGraphicsRequirementsOpenGLKHR { typedef XrResult (XRAPI_PTR *PFN_xrGetOpenGLGraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsOpenGLKHR* graphicsRequirements); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrGetOpenGLGraphicsRequirementsKHR( XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsOpenGLKHR* graphicsRequirements); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_GRAPHICS_API_OPENGL */ #ifdef XR_USE_GRAPHICS_API_OPENGL_ES @@ -162,6 +173,7 @@ XRAPI_ATTR XrResult XRAPI_CALL xrGetOpenGLGraphicsRequirementsKHR( #define XR_KHR_opengl_es_enable_SPEC_VERSION 7 #define XR_KHR_OPENGL_ES_ENABLE_EXTENSION_NAME "XR_KHR_opengl_es_enable" #ifdef XR_USE_PLATFORM_ANDROID +// XrGraphicsBindingOpenGLESAndroidKHR extends XrSessionCreateInfo typedef struct XrGraphicsBindingOpenGLESAndroidKHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -187,18 +199,21 @@ typedef struct XrGraphicsRequirementsOpenGLESKHR { typedef XrResult (XRAPI_PTR *PFN_xrGetOpenGLESGraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsOpenGLESKHR* graphicsRequirements); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrGetOpenGLESGraphicsRequirementsKHR( XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsOpenGLESKHR* graphicsRequirements); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_GRAPHICS_API_OPENGL_ES */ #ifdef XR_USE_GRAPHICS_API_VULKAN #define XR_KHR_vulkan_enable 1 -#define XR_KHR_vulkan_enable_SPEC_VERSION 7 +#define XR_KHR_vulkan_enable_SPEC_VERSION 8 #define XR_KHR_VULKAN_ENABLE_EXTENSION_NAME "XR_KHR_vulkan_enable" +// XrGraphicsBindingVulkanKHR extends XrSessionCreateInfo typedef struct XrGraphicsBindingVulkanKHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -228,6 +243,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrGetVulkanGraphicsDeviceKHR)(XrInstance instan typedef XrResult (XRAPI_PTR *PFN_xrGetVulkanGraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsVulkanKHR* graphicsRequirements); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanInstanceExtensionsKHR( XrInstance instance, XrSystemId systemId, @@ -252,7 +268,8 @@ XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanGraphicsRequirementsKHR( XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsVulkanKHR* graphicsRequirements); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_GRAPHICS_API_VULKAN */ #ifdef XR_USE_GRAPHICS_API_D3D11 @@ -260,6 +277,7 @@ XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanGraphicsRequirementsKHR( #define XR_KHR_D3D11_enable 1 #define XR_KHR_D3D11_enable_SPEC_VERSION 5 #define XR_KHR_D3D11_ENABLE_EXTENSION_NAME "XR_KHR_D3D11_enable" +// XrGraphicsBindingD3D11KHR extends XrSessionCreateInfo typedef struct XrGraphicsBindingD3D11KHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -282,11 +300,13 @@ typedef struct XrGraphicsRequirementsD3D11KHR { typedef XrResult (XRAPI_PTR *PFN_xrGetD3D11GraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsD3D11KHR* graphicsRequirements); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrGetD3D11GraphicsRequirementsKHR( XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsD3D11KHR* graphicsRequirements); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_GRAPHICS_API_D3D11 */ #ifdef XR_USE_GRAPHICS_API_D3D12 @@ -294,6 +314,7 @@ XRAPI_ATTR XrResult XRAPI_CALL xrGetD3D11GraphicsRequirementsKHR( #define XR_KHR_D3D12_enable 1 #define XR_KHR_D3D12_enable_SPEC_VERSION 7 #define XR_KHR_D3D12_ENABLE_EXTENSION_NAME "XR_KHR_D3D12_enable" +// XrGraphicsBindingD3D12KHR extends XrSessionCreateInfo typedef struct XrGraphicsBindingD3D12KHR { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -317,11 +338,13 @@ typedef struct XrGraphicsRequirementsD3D12KHR { typedef XrResult (XRAPI_PTR *PFN_xrGetD3D12GraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsD3D12KHR* graphicsRequirements); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrGetD3D12GraphicsRequirementsKHR( XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsD3D12KHR* graphicsRequirements); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_GRAPHICS_API_D3D12 */ #ifdef XR_USE_PLATFORM_WIN32 @@ -333,6 +356,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrConvertWin32PerformanceCounterToTimeKHR)(XrIn typedef XrResult (XRAPI_PTR *PFN_xrConvertTimeToWin32PerformanceCounterKHR)(XrInstance instance, XrTime time, LARGE_INTEGER* performanceCounter); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrConvertWin32PerformanceCounterToTimeKHR( XrInstance instance, const LARGE_INTEGER* performanceCounter, @@ -342,7 +366,8 @@ XRAPI_ATTR XrResult XRAPI_CALL xrConvertTimeToWin32PerformanceCounterKHR( XrInstance instance, XrTime time, LARGE_INTEGER* performanceCounter); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_PLATFORM_WIN32 */ #ifdef XR_USE_TIMESPEC @@ -354,6 +379,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrConvertTimespecTimeToTimeKHR)(XrInstance inst typedef XrResult (XRAPI_PTR *PFN_xrConvertTimeToTimespecTimeKHR)(XrInstance instance, XrTime time, struct timespec* timespecTime); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrConvertTimespecTimeToTimeKHR( XrInstance instance, const struct timespec* timespecTime, @@ -363,7 +389,8 @@ XRAPI_ATTR XrResult XRAPI_CALL xrConvertTimeToTimespecTimeKHR( XrInstance instance, XrTime time, struct timespec* timespecTime); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_TIMESPEC */ #ifdef XR_USE_PLATFORM_ANDROID @@ -383,7 +410,7 @@ typedef struct XrLoaderInitInfoAndroidKHR { #ifdef XR_USE_GRAPHICS_API_VULKAN #define XR_KHR_vulkan_enable2 1 -#define XR_KHR_vulkan_enable2_SPEC_VERSION 1 +#define XR_KHR_vulkan_enable2_SPEC_VERSION 2 #define XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME "XR_KHR_vulkan_enable2" typedef XrFlags64 XrVulkanInstanceCreateFlagsKHR; @@ -433,6 +460,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrGetVulkanGraphicsDevice2KHR)(XrInstance typedef XrResult (XRAPI_PTR *PFN_xrGetVulkanGraphicsRequirements2KHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsVulkanKHR* graphicsRequirements); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrCreateVulkanInstanceKHR( XrInstance instance, const XrVulkanInstanceCreateInfoKHR* createInfo, @@ -454,7 +482,8 @@ XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanGraphicsRequirements2KHR( XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsVulkanKHR* graphicsRequirements); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_GRAPHICS_API_VULKAN */ #ifdef XR_USE_PLATFORM_EGL @@ -462,6 +491,7 @@ XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanGraphicsRequirements2KHR( #define XR_MNDX_egl_enable 1 #define XR_MNDX_egl_enable_SPEC_VERSION 1 #define XR_MNDX_EGL_ENABLE_EXTENSION_NAME "XR_MNDX_egl_enable" +// XrGraphicsBindingEGLMNDX extends XrSessionCreateInfo typedef struct XrGraphicsBindingEGLMNDX { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -482,6 +512,7 @@ typedef XrResult (XRAPI_PTR *PFN_xrCreateSpatialAnchorFromPerceptionAnchorMSFT)( typedef XrResult (XRAPI_PTR *PFN_xrTryGetPerceptionAnchorFromSpatialAnchorMSFT)(XrSession session, XrSpatialAnchorMSFT anchor, IUnknown** perceptionAnchor); #ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES XRAPI_ATTR XrResult XRAPI_CALL xrCreateSpatialAnchorFromPerceptionAnchorMSFT( XrSession session, IUnknown* perceptionAnchor, @@ -491,7 +522,8 @@ XRAPI_ATTR XrResult XRAPI_CALL xrTryGetPerceptionAnchorFromSpatialAnchorMSFT( XrSession session, XrSpatialAnchorMSFT anchor, IUnknown** perceptionAnchor); -#endif +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ #endif /* XR_USE_PLATFORM_WIN32 */ #ifdef XR_USE_PLATFORM_WIN32 @@ -500,6 +532,7 @@ XRAPI_ATTR XrResult XRAPI_CALL xrTryGetPerceptionAnchorFromSpatialAnchorMSFT( #define XR_MSFT_holographic_window_attachment_SPEC_VERSION 1 #define XR_MSFT_HOLOGRAPHIC_WINDOW_ATTACHMENT_EXTENSION_NAME "XR_MSFT_holographic_window_attachment" #ifdef XR_USE_PLATFORM_WIN32 +// XrHolographicWindowAttachmentMSFT extends XrSessionCreateInfo typedef struct XrHolographicWindowAttachmentMSFT { XrStructureType type; const void* XR_MAY_ALIAS next; @@ -510,6 +543,131 @@ typedef struct XrHolographicWindowAttachmentMSFT { #endif /* XR_USE_PLATFORM_WIN32 */ +#ifdef XR_USE_PLATFORM_ANDROID + +#define XR_FB_android_surface_swapchain_create 1 +#define XR_FB_android_surface_swapchain_create_SPEC_VERSION 1 +#define XR_FB_ANDROID_SURFACE_SWAPCHAIN_CREATE_EXTENSION_NAME "XR_FB_android_surface_swapchain_create" +typedef XrFlags64 XrAndroidSurfaceSwapchainFlagsFB; + +// Flag bits for XrAndroidSurfaceSwapchainFlagsFB +static const XrAndroidSurfaceSwapchainFlagsFB XR_ANDROID_SURFACE_SWAPCHAIN_SYNCHRONOUS_BIT_FB = 0x00000001; +static const XrAndroidSurfaceSwapchainFlagsFB XR_ANDROID_SURFACE_SWAPCHAIN_USE_TIMESTAMPS_BIT_FB = 0x00000002; + +#ifdef XR_USE_PLATFORM_ANDROID +// XrAndroidSurfaceSwapchainCreateInfoFB extends XrSwapchainCreateInfo +typedef struct XrAndroidSurfaceSwapchainCreateInfoFB { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrAndroidSurfaceSwapchainFlagsFB createFlags; +} XrAndroidSurfaceSwapchainCreateInfoFB; +#endif // XR_USE_PLATFORM_ANDROID + +#endif /* XR_USE_PLATFORM_ANDROID */ + +#ifdef XR_USE_PLATFORM_WIN32 + +#define XR_OCULUS_audio_device_guid 1 +#define XR_OCULUS_audio_device_guid_SPEC_VERSION 1 +#define XR_OCULUS_AUDIO_DEVICE_GUID_EXTENSION_NAME "XR_OCULUS_audio_device_guid" +#define XR_MAX_AUDIO_DEVICE_STR_SIZE_OCULUS 128 +typedef XrResult (XRAPI_PTR *PFN_xrGetAudioOutputDeviceGuidOculus)(XrInstance instance, wchar_t buffer[XR_MAX_AUDIO_DEVICE_STR_SIZE_OCULUS]); +typedef XrResult (XRAPI_PTR *PFN_xrGetAudioInputDeviceGuidOculus)(XrInstance instance, wchar_t buffer[XR_MAX_AUDIO_DEVICE_STR_SIZE_OCULUS]); + +#ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES +XRAPI_ATTR XrResult XRAPI_CALL xrGetAudioOutputDeviceGuidOculus( + XrInstance instance, + wchar_t buffer[XR_MAX_AUDIO_DEVICE_STR_SIZE_OCULUS]); + +XRAPI_ATTR XrResult XRAPI_CALL xrGetAudioInputDeviceGuidOculus( + XrInstance instance, + wchar_t buffer[XR_MAX_AUDIO_DEVICE_STR_SIZE_OCULUS]); +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ +#endif /* XR_USE_PLATFORM_WIN32 */ + +#ifdef XR_USE_GRAPHICS_API_VULKAN + +#define XR_FB_foveation_vulkan 1 +#define XR_FB_foveation_vulkan_SPEC_VERSION 1 +#define XR_FB_FOVEATION_VULKAN_EXTENSION_NAME "XR_FB_foveation_vulkan" +// XrSwapchainImageFoveationVulkanFB extends XrSwapchainImageVulkanKHR +typedef struct XrSwapchainImageFoveationVulkanFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + VkImage image; + uint32_t width; + uint32_t height; +} XrSwapchainImageFoveationVulkanFB; + +#endif /* XR_USE_GRAPHICS_API_VULKAN */ + +#ifdef XR_USE_PLATFORM_ANDROID + +#define XR_FB_swapchain_update_state_android_surface 1 +#define XR_FB_swapchain_update_state_android_surface_SPEC_VERSION 1 +#define XR_FB_SWAPCHAIN_UPDATE_STATE_ANDROID_SURFACE_EXTENSION_NAME "XR_FB_swapchain_update_state_android_surface" +#ifdef XR_USE_PLATFORM_ANDROID +typedef struct XrSwapchainStateAndroidSurfaceDimensionsFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint32_t width; + uint32_t height; +} XrSwapchainStateAndroidSurfaceDimensionsFB; +#endif // XR_USE_PLATFORM_ANDROID + +#endif /* XR_USE_PLATFORM_ANDROID */ + +#ifdef XR_USE_GRAPHICS_API_OPENGL_ES + +#define XR_FB_swapchain_update_state_opengl_es 1 +#define XR_FB_swapchain_update_state_opengl_es_SPEC_VERSION 1 +#define XR_FB_SWAPCHAIN_UPDATE_STATE_OPENGL_ES_EXTENSION_NAME "XR_FB_swapchain_update_state_opengl_es" +#ifdef XR_USE_GRAPHICS_API_OPENGL_ES +typedef struct XrSwapchainStateSamplerOpenGLESFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + EGLenum minFilter; + EGLenum magFilter; + EGLenum wrapModeS; + EGLenum wrapModeT; + EGLenum swizzleRed; + EGLenum swizzleGreen; + EGLenum swizzleBlue; + EGLenum swizzleAlpha; + float maxAnisotropy; + XrColor4f borderColor; +} XrSwapchainStateSamplerOpenGLESFB; +#endif // XR_USE_GRAPHICS_API_OPENGL_ES + +#endif /* XR_USE_GRAPHICS_API_OPENGL_ES */ + +#ifdef XR_USE_GRAPHICS_API_VULKAN + +#define XR_FB_swapchain_update_state_vulkan 1 +#define XR_FB_swapchain_update_state_vulkan_SPEC_VERSION 1 +#define XR_FB_SWAPCHAIN_UPDATE_STATE_VULKAN_EXTENSION_NAME "XR_FB_swapchain_update_state_vulkan" +#ifdef XR_USE_GRAPHICS_API_VULKAN +typedef struct XrSwapchainStateSamplerVulkanFB { + XrStructureType type; + void* XR_MAY_ALIAS next; + VkFilter minFilter; + VkFilter magFilter; + VkSamplerMipmapMode mipmapMode; + VkSamplerAddressMode wrapModeS; + VkSamplerAddressMode wrapModeT; + VkComponentSwizzle swizzleRed; + VkComponentSwizzle swizzleGreen; + VkComponentSwizzle swizzleBlue; + VkComponentSwizzle swizzleAlpha; + float maxAnisotropy; + XrColor4f borderColor; +} XrSwapchainStateSamplerVulkanFB; +#endif // XR_USE_GRAPHICS_API_VULKAN + +#endif /* XR_USE_GRAPHICS_API_VULKAN */ + #ifdef __cplusplus } #endif diff --git a/src/external/openxr_includes/openxr/openxr_platform_defines.h b/src/external/openxr_includes/openxr/openxr_platform_defines.h index 45ccf0e37..a7ffcb459 100644 --- a/src/external/openxr_includes/openxr/openxr_platform_defines.h +++ b/src/external/openxr_includes/openxr/openxr_platform_defines.h @@ -1,5 +1,5 @@ /* -** Copyright (c) 2017-2020 The Khronos Group Inc. +** Copyright (c) 2017-2021, The Khronos Group Inc. ** ** SPDX-License-Identifier: Apache-2.0 OR MIT */ diff --git a/src/external/openxr_includes/openxr/openxr_reflection.h b/src/external/openxr_includes/openxr/openxr_reflection.h index 7249f72e8..7d9cdda21 100644 --- a/src/external/openxr_includes/openxr/openxr_reflection.h +++ b/src/external/openxr_includes/openxr/openxr_reflection.h @@ -2,7 +2,7 @@ #define OPENXR_REFLECTION_H_ 1 /* -** Copyright (c) 2017-2020 The Khronos Group Inc. +** Copyright (c) 2017-2021, The Khronos Group Inc. ** ** SPDX-License-Identifier: Apache-2.0 OR MIT */ @@ -85,13 +85,23 @@ XR_ENUM_STR(XrResult); _(XR_ERROR_LOCALIZED_NAME_DUPLICATED, -48) \ _(XR_ERROR_LOCALIZED_NAME_INVALID, -49) \ _(XR_ERROR_GRAPHICS_REQUIREMENTS_CALL_MISSING, -50) \ + _(XR_ERROR_RUNTIME_UNAVAILABLE, -51) \ _(XR_ERROR_ANDROID_THREAD_SETTINGS_ID_INVALID_KHR, -1000003000) \ _(XR_ERROR_ANDROID_THREAD_SETTINGS_FAILURE_KHR, -1000003001) \ _(XR_ERROR_CREATE_SPATIAL_ANCHOR_FAILED_MSFT, -1000039001) \ _(XR_ERROR_SECONDARY_VIEW_CONFIGURATION_TYPE_NOT_ENABLED_MSFT, -1000053000) \ _(XR_ERROR_CONTROLLER_MODEL_KEY_INVALID_MSFT, -1000055000) \ + _(XR_ERROR_REPROJECTION_MODE_UNSUPPORTED_MSFT, -1000066000) \ + _(XR_ERROR_COMPUTE_NEW_SCENE_NOT_COMPLETED_MSFT, -1000097000) \ + _(XR_ERROR_SCENE_COMPONENT_ID_INVALID_MSFT, -1000097001) \ + _(XR_ERROR_SCENE_COMPONENT_TYPE_MISMATCH_MSFT, -1000097002) \ + _(XR_ERROR_SCENE_MESH_BUFFER_ID_INVALID_MSFT, -1000097003) \ + _(XR_ERROR_SCENE_COMPUTE_FEATURE_INCOMPATIBLE_MSFT, -1000097004) \ + _(XR_ERROR_SCENE_COMPUTE_CONSISTENCY_MISMATCH_MSFT, -1000097005) \ _(XR_ERROR_DISPLAY_REFRESH_RATE_UNSUPPORTED_FB, -1000101000) \ _(XR_ERROR_COLOR_SPACE_UNSUPPORTED_FB, -1000108000) \ + _(XR_ERROR_SPATIAL_ANCHOR_NAME_NOT_FOUND_MSFT, -1000142001) \ + _(XR_ERROR_SPATIAL_ANCHOR_NAME_INVALID_MSFT, -1000142002) \ _(XR_RESULT_MAX_ENUM, 0x7FFFFFFF) #define XR_LIST_ENUM_XrStructureType(_) \ @@ -183,6 +193,8 @@ XR_ENUM_STR(XrResult); _(XR_TYPE_COMPOSITION_LAYER_COLOR_SCALE_BIAS_KHR, 1000034000) \ _(XR_TYPE_SPATIAL_ANCHOR_CREATE_INFO_MSFT, 1000039000) \ _(XR_TYPE_SPATIAL_ANCHOR_SPACE_CREATE_INFO_MSFT, 1000039001) \ + _(XR_TYPE_COMPOSITION_LAYER_IMAGE_LAYOUT_FB, 1000040000) \ + _(XR_TYPE_COMPOSITION_LAYER_ALPHA_BLEND_FB, 1000041001) \ _(XR_TYPE_VIEW_CONFIGURATION_DEPTH_RANGE_EXT, 1000046000) \ _(XR_TYPE_GRAPHICS_BINDING_EGL_MNDX, 1000048004) \ _(XR_TYPE_SPATIAL_GRAPH_NODE_SPACE_CREATE_INFO_MSFT, 1000049000) \ @@ -209,15 +221,61 @@ XR_ENUM_STR(XrResult); _(XR_TYPE_CONTROLLER_MODEL_STATE_MSFT, 1000055004) \ _(XR_TYPE_VIEW_CONFIGURATION_VIEW_FOV_EPIC, 1000059000) \ _(XR_TYPE_HOLOGRAPHIC_WINDOW_ATTACHMENT_MSFT, 1000063000) \ + _(XR_TYPE_COMPOSITION_LAYER_REPROJECTION_INFO_MSFT, 1000066000) \ + _(XR_TYPE_COMPOSITION_LAYER_REPROJECTION_PLANE_OVERRIDE_MSFT, 1000066001) \ + _(XR_TYPE_ANDROID_SURFACE_SWAPCHAIN_CREATE_INFO_FB, 1000070000) \ + _(XR_TYPE_COMPOSITION_LAYER_SECURE_CONTENT_FB, 1000072000) \ _(XR_TYPE_INTERACTION_PROFILE_ANALOG_THRESHOLD_VALVE, 1000079000) \ + _(XR_TYPE_HAND_JOINTS_MOTION_RANGE_INFO_EXT, 1000080000) \ _(XR_TYPE_LOADER_INIT_INFO_ANDROID_KHR, 1000089000) \ _(XR_TYPE_VULKAN_INSTANCE_CREATE_INFO_KHR, 1000090000) \ _(XR_TYPE_VULKAN_DEVICE_CREATE_INFO_KHR, 1000090001) \ _(XR_TYPE_VULKAN_GRAPHICS_DEVICE_GET_INFO_KHR, 1000090003) \ _(XR_TYPE_COMPOSITION_LAYER_EQUIRECT2_KHR, 1000091000) \ + _(XR_TYPE_SCENE_OBSERVER_CREATE_INFO_MSFT, 1000097000) \ + _(XR_TYPE_SCENE_CREATE_INFO_MSFT, 1000097001) \ + _(XR_TYPE_NEW_SCENE_COMPUTE_INFO_MSFT, 1000097002) \ + _(XR_TYPE_VISUAL_MESH_COMPUTE_LOD_INFO_MSFT, 1000097003) \ + _(XR_TYPE_SCENE_COMPONENTS_MSFT, 1000097004) \ + _(XR_TYPE_SCENE_COMPONENTS_GET_INFO_MSFT, 1000097005) \ + _(XR_TYPE_SCENE_COMPONENT_LOCATIONS_MSFT, 1000097006) \ + _(XR_TYPE_SCENE_COMPONENTS_LOCATE_INFO_MSFT, 1000097007) \ + _(XR_TYPE_SCENE_OBJECTS_MSFT, 1000097008) \ + _(XR_TYPE_SCENE_COMPONENT_PARENT_FILTER_INFO_MSFT, 1000097009) \ + _(XR_TYPE_SCENE_OBJECT_TYPES_FILTER_INFO_MSFT, 1000097010) \ + _(XR_TYPE_SCENE_PLANES_MSFT, 1000097011) \ + _(XR_TYPE_SCENE_PLANE_ALIGNMENT_FILTER_INFO_MSFT, 1000097012) \ + _(XR_TYPE_SCENE_MESHES_MSFT, 1000097013) \ + _(XR_TYPE_SCENE_MESH_BUFFERS_GET_INFO_MSFT, 1000097014) \ + _(XR_TYPE_SCENE_MESH_BUFFERS_MSFT, 1000097015) \ + _(XR_TYPE_SCENE_MESH_VERTEX_BUFFER_MSFT, 1000097016) \ + _(XR_TYPE_SCENE_MESH_INDICES_UINT32_MSFT, 1000097017) \ + _(XR_TYPE_SCENE_MESH_INDICES_UINT16_MSFT, 1000097018) \ + _(XR_TYPE_SERIALIZED_SCENE_FRAGMENT_DATA_GET_INFO_MSFT, 1000098000) \ + _(XR_TYPE_SCENE_DESERIALIZE_INFO_MSFT, 1000098001) \ _(XR_TYPE_EVENT_DATA_DISPLAY_REFRESH_RATE_CHANGED_FB, 1000101000) \ _(XR_TYPE_SYSTEM_COLOR_SPACE_PROPERTIES_FB, 1000108000) \ + _(XR_TYPE_HAND_TRACKING_MESH_FB, 1000110001) \ + _(XR_TYPE_HAND_TRACKING_SCALE_FB, 1000110003) \ + _(XR_TYPE_HAND_TRACKING_AIM_STATE_FB, 1000111001) \ + _(XR_TYPE_HAND_TRACKING_CAPSULES_STATE_FB, 1000112000) \ + _(XR_TYPE_FOVEATION_PROFILE_CREATE_INFO_FB, 1000114000) \ + _(XR_TYPE_SWAPCHAIN_CREATE_INFO_FOVEATION_FB, 1000114001) \ + _(XR_TYPE_SWAPCHAIN_STATE_FOVEATION_FB, 1000114002) \ + _(XR_TYPE_FOVEATION_LEVEL_PROFILE_CREATE_INFO_FB, 1000115000) \ _(XR_TYPE_BINDING_MODIFICATIONS_KHR, 1000120000) \ + _(XR_TYPE_VIEW_LOCATE_FOVEATED_RENDERING_VARJO, 1000121000) \ + _(XR_TYPE_FOVEATED_VIEW_CONFIGURATION_VIEW_VARJO, 1000121001) \ + _(XR_TYPE_SYSTEM_FOVEATED_RENDERING_PROPERTIES_VARJO, 1000121002) \ + _(XR_TYPE_COMPOSITION_LAYER_DEPTH_TEST_VARJO, 1000122000) \ + _(XR_TYPE_SPATIAL_ANCHOR_PERSISTENCE_INFO_MSFT, 1000142000) \ + _(XR_TYPE_SPATIAL_ANCHOR_FROM_PERSISTED_ANCHOR_CREATE_INFO_MSFT, 1000142001) \ + _(XR_TYPE_SWAPCHAIN_IMAGE_FOVEATION_VULKAN_FB, 1000160000) \ + _(XR_TYPE_SWAPCHAIN_STATE_ANDROID_SURFACE_DIMENSIONS_FB, 1000161000) \ + _(XR_TYPE_SWAPCHAIN_STATE_SAMPLER_OPENGL_ES_FB, 1000162000) \ + _(XR_TYPE_SWAPCHAIN_STATE_SAMPLER_VULKAN_FB, 1000163000) \ + _(XR_TYPE_COMPOSITION_LAYER_SPACE_WARP_INFO_FB, 1000171000) \ + _(XR_TYPE_SYSTEM_SPACE_WARP_PROPERTIES_FB, 1000171001) \ _(XR_STRUCTURE_TYPE_MAX_ENUM, 0x7FFFFFFF) #define XR_LIST_ENUM_XrFormFactor(_) \ @@ -243,6 +301,7 @@ XR_ENUM_STR(XrResult); _(XR_REFERENCE_SPACE_TYPE_LOCAL, 2) \ _(XR_REFERENCE_SPACE_TYPE_STAGE, 3) \ _(XR_REFERENCE_SPACE_TYPE_UNBOUNDED_MSFT, 1000038000) \ + _(XR_REFERENCE_SPACE_TYPE_COMBINED_EYE_VARJO, 1000121000) \ _(XR_REFERENCE_SPACE_TYPE_MAX_ENUM, 0x7FFFFFFF) #define XR_LIST_ENUM_XrActionType(_) \ @@ -282,6 +341,10 @@ XR_ENUM_STR(XrResult); _(XR_OBJECT_TYPE_DEBUG_UTILS_MESSENGER_EXT, 1000019000) \ _(XR_OBJECT_TYPE_SPATIAL_ANCHOR_MSFT, 1000039000) \ _(XR_OBJECT_TYPE_HAND_TRACKER_EXT, 1000051000) \ + _(XR_OBJECT_TYPE_SCENE_OBSERVER_MSFT, 1000097000) \ + _(XR_OBJECT_TYPE_SCENE_MSFT, 1000097001) \ + _(XR_OBJECT_TYPE_FOVEATION_PROFILE_FB, 1000114000) \ + _(XR_OBJECT_TYPE_SPATIAL_ANCHOR_STORE_CONNECTION_MSFT, 1000142000) \ _(XR_OBJECT_TYPE_MAX_ENUM, 0x7FFFFFFF) #define XR_LIST_ENUM_XrAndroidThreadTypeKHR(_) \ @@ -321,6 +384,15 @@ XR_ENUM_STR(XrResult); _(XR_PERF_SETTINGS_NOTIF_LEVEL_IMPAIRED_EXT, 75) \ _(XR_PERF_SETTINGS_NOTIFICATION_LEVEL_MAX_ENUM_EXT, 0x7FFFFFFF) +#define XR_LIST_ENUM_XrBlendFactorFB(_) \ + _(XR_BLEND_FACTOR_ZERO_FB, 0) \ + _(XR_BLEND_FACTOR_ONE_FB, 1) \ + _(XR_BLEND_FACTOR_SRC_ALPHA_FB, 2) \ + _(XR_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA_FB, 3) \ + _(XR_BLEND_FACTOR_DST_ALPHA_FB, 4) \ + _(XR_BLEND_FACTOR_ONE_MINUS_DST_ALPHA_FB, 5) \ + _(XR_BLEND_FACTOR_MAX_ENUM_FB, 0x7FFFFFFF) + #define XR_LIST_ENUM_XrSpatialGraphNodeTypeMSFT(_) \ _(XR_SPATIAL_GRAPH_NODE_TYPE_STATIC_MSFT, 1) \ _(XR_SPATIAL_GRAPH_NODE_TYPE_DYNAMIC_MSFT, 2) \ @@ -369,6 +441,71 @@ XR_ENUM_STR(XrResult); _(XR_HAND_POSE_TYPE_REFERENCE_OPEN_PALM_MSFT, 1) \ _(XR_HAND_POSE_TYPE_MAX_ENUM_MSFT, 0x7FFFFFFF) +#define XR_LIST_ENUM_XrReprojectionModeMSFT(_) \ + _(XR_REPROJECTION_MODE_DEPTH_MSFT, 1) \ + _(XR_REPROJECTION_MODE_PLANAR_FROM_DEPTH_MSFT, 2) \ + _(XR_REPROJECTION_MODE_PLANAR_MANUAL_MSFT, 3) \ + _(XR_REPROJECTION_MODE_ORIENTATION_ONLY_MSFT, 4) \ + _(XR_REPROJECTION_MODE_MAX_ENUM_MSFT, 0x7FFFFFFF) + +#define XR_LIST_ENUM_XrHandJointsMotionRangeEXT(_) \ + _(XR_HAND_JOINTS_MOTION_RANGE_UNOBSTRUCTED_EXT, 1) \ + _(XR_HAND_JOINTS_MOTION_RANGE_CONFORMING_TO_CONTROLLER_EXT, 2) \ + _(XR_HAND_JOINTS_MOTION_RANGE_MAX_ENUM_EXT, 0x7FFFFFFF) + +#define XR_LIST_ENUM_XrSceneComputeFeatureMSFT(_) \ + _(XR_SCENE_COMPUTE_FEATURE_PLANE_MSFT, 1) \ + _(XR_SCENE_COMPUTE_FEATURE_PLANE_MESH_MSFT, 2) \ + _(XR_SCENE_COMPUTE_FEATURE_VISUAL_MESH_MSFT, 3) \ + _(XR_SCENE_COMPUTE_FEATURE_COLLIDER_MESH_MSFT, 4) \ + _(XR_SCENE_COMPUTE_FEATURE_SERIALIZE_SCENE_MSFT, 1000098000) \ + _(XR_SCENE_COMPUTE_FEATURE_MAX_ENUM_MSFT, 0x7FFFFFFF) + +#define XR_LIST_ENUM_XrSceneComputeConsistencyMSFT(_) \ + _(XR_SCENE_COMPUTE_CONSISTENCY_SNAPSHOT_COMPLETE_MSFT, 1) \ + _(XR_SCENE_COMPUTE_CONSISTENCY_SNAPSHOT_INCOMPLETE_FAST_MSFT, 2) \ + _(XR_SCENE_COMPUTE_CONSISTENCY_OCCLUSION_OPTIMIZED_MSFT, 3) \ + _(XR_SCENE_COMPUTE_CONSISTENCY_MAX_ENUM_MSFT, 0x7FFFFFFF) + +#define XR_LIST_ENUM_XrMeshComputeLodMSFT(_) \ + _(XR_MESH_COMPUTE_LOD_COARSE_MSFT, 1) \ + _(XR_MESH_COMPUTE_LOD_MEDIUM_MSFT, 2) \ + _(XR_MESH_COMPUTE_LOD_FINE_MSFT, 3) \ + _(XR_MESH_COMPUTE_LOD_UNLIMITED_MSFT, 4) \ + _(XR_MESH_COMPUTE_LOD_MAX_ENUM_MSFT, 0x7FFFFFFF) + +#define XR_LIST_ENUM_XrSceneComponentTypeMSFT(_) \ + _(XR_SCENE_COMPONENT_TYPE_INVALID_MSFT, -1) \ + _(XR_SCENE_COMPONENT_TYPE_OBJECT_MSFT, 1) \ + _(XR_SCENE_COMPONENT_TYPE_PLANE_MSFT, 2) \ + _(XR_SCENE_COMPONENT_TYPE_VISUAL_MESH_MSFT, 3) \ + _(XR_SCENE_COMPONENT_TYPE_COLLIDER_MESH_MSFT, 4) \ + _(XR_SCENE_COMPONENT_TYPE_SERIALIZED_SCENE_FRAGMENT_MSFT, 1000098000) \ + _(XR_SCENE_COMPONENT_TYPE_MAX_ENUM_MSFT, 0x7FFFFFFF) + +#define XR_LIST_ENUM_XrSceneObjectTypeMSFT(_) \ + _(XR_SCENE_OBJECT_TYPE_UNCATEGORIZED_MSFT, -1) \ + _(XR_SCENE_OBJECT_TYPE_BACKGROUND_MSFT, 1) \ + _(XR_SCENE_OBJECT_TYPE_WALL_MSFT, 2) \ + _(XR_SCENE_OBJECT_TYPE_FLOOR_MSFT, 3) \ + _(XR_SCENE_OBJECT_TYPE_CEILING_MSFT, 4) \ + _(XR_SCENE_OBJECT_TYPE_PLATFORM_MSFT, 5) \ + _(XR_SCENE_OBJECT_TYPE_INFERRED_MSFT, 6) \ + _(XR_SCENE_OBJECT_TYPE_MAX_ENUM_MSFT, 0x7FFFFFFF) + +#define XR_LIST_ENUM_XrScenePlaneAlignmentTypeMSFT(_) \ + _(XR_SCENE_PLANE_ALIGNMENT_TYPE_NON_ORTHOGONAL_MSFT, 0) \ + _(XR_SCENE_PLANE_ALIGNMENT_TYPE_HORIZONTAL_MSFT, 1) \ + _(XR_SCENE_PLANE_ALIGNMENT_TYPE_VERTICAL_MSFT, 2) \ + _(XR_SCENE_PLANE_ALIGNMENT_TYPE_MAX_ENUM_MSFT, 0x7FFFFFFF) + +#define XR_LIST_ENUM_XrSceneComputeStateMSFT(_) \ + _(XR_SCENE_COMPUTE_STATE_NONE_MSFT, 0) \ + _(XR_SCENE_COMPUTE_STATE_UPDATING_MSFT, 1) \ + _(XR_SCENE_COMPUTE_STATE_COMPLETED_MSFT, 2) \ + _(XR_SCENE_COMPUTE_STATE_COMPLETED_WITH_ERROR_MSFT, 3) \ + _(XR_SCENE_COMPUTE_STATE_MAX_ENUM_MSFT, 0x7FFFFFFF) + #define XR_LIST_ENUM_XrColorSpaceFB(_) \ _(XR_COLOR_SPACE_UNMANAGED_FB, 0) \ _(XR_COLOR_SPACE_REC2020_FB, 1) \ @@ -380,6 +517,18 @@ XR_ENUM_STR(XrResult); _(XR_COLOR_SPACE_ADOBE_RGB_FB, 7) \ _(XR_COLOR_SPACE_MAX_ENUM_FB, 0x7FFFFFFF) +#define XR_LIST_ENUM_XrFoveationLevelFB(_) \ + _(XR_FOVEATION_LEVEL_NONE_FB, 0) \ + _(XR_FOVEATION_LEVEL_LOW_FB, 1) \ + _(XR_FOVEATION_LEVEL_MEDIUM_FB, 2) \ + _(XR_FOVEATION_LEVEL_HIGH_FB, 3) \ + _(XR_FOVEATION_LEVEL_MAX_ENUM_FB, 0x7FFFFFFF) + +#define XR_LIST_ENUM_XrFoveationDynamicFB(_) \ + _(XR_FOVEATION_DYNAMIC_DISABLED_FB, 0) \ + _(XR_FOVEATION_DYNAMIC_LEVEL_ENABLED_FB, 1) \ + _(XR_FOVEATION_DYNAMIC_MAX_ENUM_FB, 0x7FFFFFFF) + #define XR_LIST_BITS_XrInstanceCreateFlags(_) #define XR_LIST_BITS_XrSessionCreateFlags(_) @@ -407,6 +556,7 @@ XR_ENUM_STR(XrResult); _(XR_SWAPCHAIN_USAGE_SAMPLED_BIT, 0x00000020) \ _(XR_SWAPCHAIN_USAGE_MUTABLE_FORMAT_BIT, 0x00000040) \ _(XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_MND, 0x00000080) \ + _(XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_KHR, XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_MND) \ #define XR_LIST_BITS_XrCompositionLayerFlags(_) \ _(XR_COMPOSITION_LAYER_CORRECT_CHROMATIC_ABERRATION_BIT, 0x00000001) \ @@ -440,12 +590,41 @@ XR_ENUM_STR(XrResult); _(XR_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT, 0x00000004) \ _(XR_DEBUG_UTILS_MESSAGE_TYPE_CONFORMANCE_BIT_EXT, 0x00000008) \ -#define XR_LIST_BITS_XrOverlaySessionCreateFlagsEXTX(_) \ - _(XR_OVERLAY_SESSION_CREATE_RELAXED_DISPLAY_TIME_BIT_EXTX, 0x00000001) \ +#define XR_LIST_BITS_XrOverlaySessionCreateFlagsEXTX(_) #define XR_LIST_BITS_XrOverlayMainSessionFlagsEXTX(_) \ _(XR_OVERLAY_MAIN_SESSION_ENABLED_COMPOSITION_LAYER_INFO_DEPTH_BIT_EXTX, 0x00000001) \ +#define XR_LIST_BITS_XrCompositionLayerImageLayoutFlagsFB(_) \ + _(XR_COMPOSITION_LAYER_IMAGE_LAYOUT_VERTICAL_FLIP_BIT_FB, 0x00000001) \ + +#define XR_LIST_BITS_XrAndroidSurfaceSwapchainFlagsFB(_) \ + _(XR_ANDROID_SURFACE_SWAPCHAIN_SYNCHRONOUS_BIT_FB, 0x00000001) \ + _(XR_ANDROID_SURFACE_SWAPCHAIN_USE_TIMESTAMPS_BIT_FB, 0x00000002) \ + +#define XR_LIST_BITS_XrCompositionLayerSecureContentFlagsFB(_) \ + _(XR_COMPOSITION_LAYER_SECURE_CONTENT_EXCLUDE_LAYER_BIT_FB, 0x00000001) \ + _(XR_COMPOSITION_LAYER_SECURE_CONTENT_REPLACE_LAYER_BIT_FB, 0x00000002) \ + +#define XR_LIST_BITS_XrHandTrackingAimFlagsFB(_) \ + _(XR_HAND_TRACKING_AIM_COMPUTED_BIT_FB, 0x00000001) \ + _(XR_HAND_TRACKING_AIM_VALID_BIT_FB, 0x00000002) \ + _(XR_HAND_TRACKING_AIM_INDEX_PINCHING_BIT_FB, 0x00000004) \ + _(XR_HAND_TRACKING_AIM_MIDDLE_PINCHING_BIT_FB, 0x00000008) \ + _(XR_HAND_TRACKING_AIM_RING_PINCHING_BIT_FB, 0x00000010) \ + _(XR_HAND_TRACKING_AIM_LITTLE_PINCHING_BIT_FB, 0x00000020) \ + _(XR_HAND_TRACKING_AIM_SYSTEM_GESTURE_BIT_FB, 0x00000040) \ + _(XR_HAND_TRACKING_AIM_DOMINANT_HAND_BIT_FB, 0x00000080) \ + _(XR_HAND_TRACKING_AIM_MENU_PRESSED_BIT_FB, 0x00000100) \ + +#define XR_LIST_BITS_XrSwapchainCreateFoveationFlagsFB(_) \ + _(XR_SWAPCHAIN_CREATE_FOVEATION_SCALED_BIN_BIT_FB, 0x00000001) \ + _(XR_SWAPCHAIN_CREATE_FOVEATION_FRAGMENT_DENSITY_MAP_BIT_FB, 0x00000002) \ + +#define XR_LIST_BITS_XrSwapchainStateFoveationFlagsFB(_) + +#define XR_LIST_BITS_XrCompositionLayerSpaceWarpInfoFlagsFB(_) + #define XR_LIST_STRUCT_XrApiLayerProperties(_) \ _(type) \ _(next) \ @@ -1208,6 +1387,19 @@ XR_ENUM_STR(XrResult); _(anchor) \ _(poseInAnchorSpace) \ +#define XR_LIST_STRUCT_XrCompositionLayerImageLayoutFB(_) \ + _(type) \ + _(next) \ + _(flags) \ + +#define XR_LIST_STRUCT_XrCompositionLayerAlphaBlendFB(_) \ + _(type) \ + _(next) \ + _(srcFactorColor) \ + _(dstFactorColor) \ + _(srcFactorAlpha) \ + _(dstFactorAlpha) \ + #define XR_LIST_STRUCT_XrViewConfigurationDepthRangeEXT(_) \ _(type) \ _(next) \ @@ -1399,6 +1591,32 @@ XR_ENUM_STR(XrResult); _(holographicSpace) \ _(coreWindow) \ +#define XR_LIST_STRUCT_XrCompositionLayerReprojectionInfoMSFT(_) \ + _(type) \ + _(next) \ + _(reprojectionMode) \ + +#define XR_LIST_STRUCT_XrCompositionLayerReprojectionPlaneOverrideMSFT(_) \ + _(type) \ + _(next) \ + _(position) \ + _(normal) \ + _(velocity) \ + +#define XR_LIST_STRUCT_XrAndroidSurfaceSwapchainCreateInfoFB(_) \ + _(type) \ + _(next) \ + _(createFlags) \ + +#define XR_LIST_STRUCT_XrSwapchainStateBaseHeaderFB(_) \ + _(type) \ + _(next) \ + +#define XR_LIST_STRUCT_XrCompositionLayerSecureContentFB(_) \ + _(type) \ + _(next) \ + _(flags) \ + #define XR_LIST_STRUCT_XrInteractionProfileAnalogThresholdVALVE(_) \ _(type) \ _(next) \ @@ -1409,6 +1627,187 @@ XR_ENUM_STR(XrResult); _(onHaptic) \ _(offHaptic) \ +#define XR_LIST_STRUCT_XrHandJointsMotionRangeInfoEXT(_) \ + _(type) \ + _(next) \ + _(handJointsMotionRange) \ + +#define XR_LIST_STRUCT_XrUuidMSFT(_) \ + _(bytes) \ + +#define XR_LIST_STRUCT_XrSceneObserverCreateInfoMSFT(_) \ + _(type) \ + _(next) \ + +#define XR_LIST_STRUCT_XrSceneCreateInfoMSFT(_) \ + _(type) \ + _(next) \ + +#define XR_LIST_STRUCT_XrSceneSphereBoundMSFT(_) \ + _(center) \ + _(radius) \ + +#define XR_LIST_STRUCT_XrSceneOrientedBoxBoundMSFT(_) \ + _(pose) \ + _(extents) \ + +#define XR_LIST_STRUCT_XrSceneFrustumBoundMSFT(_) \ + _(pose) \ + _(fov) \ + _(farDistance) \ + +#define XR_LIST_STRUCT_XrSceneBoundsMSFT(_) \ + _(space) \ + _(time) \ + _(sphereCount) \ + _(spheres) \ + _(boxCount) \ + _(boxes) \ + _(frustumCount) \ + _(frustums) \ + +#define XR_LIST_STRUCT_XrNewSceneComputeInfoMSFT(_) \ + _(type) \ + _(next) \ + _(requestedFeatureCount) \ + _(requestedFeatures) \ + _(consistency) \ + _(bounds) \ + +#define XR_LIST_STRUCT_XrVisualMeshComputeLodInfoMSFT(_) \ + _(type) \ + _(next) \ + _(lod) \ + +#define XR_LIST_STRUCT_XrSceneComponentMSFT(_) \ + _(componentType) \ + _(id) \ + _(parentId) \ + _(updateTime) \ + +#define XR_LIST_STRUCT_XrSceneComponentsMSFT(_) \ + _(type) \ + _(next) \ + _(componentCapacityInput) \ + _(componentCountOutput) \ + _(components) \ + +#define XR_LIST_STRUCT_XrSceneComponentsGetInfoMSFT(_) \ + _(type) \ + _(next) \ + _(componentType) \ + +#define XR_LIST_STRUCT_XrSceneComponentLocationMSFT(_) \ + _(flags) \ + _(pose) \ + +#define XR_LIST_STRUCT_XrSceneComponentLocationsMSFT(_) \ + _(type) \ + _(next) \ + _(locationCount) \ + _(locations) \ + +#define XR_LIST_STRUCT_XrSceneComponentsLocateInfoMSFT(_) \ + _(type) \ + _(next) \ + _(baseSpace) \ + _(time) \ + _(componentIdCount) \ + _(componentIds) \ + +#define XR_LIST_STRUCT_XrSceneObjectMSFT(_) \ + _(objectType) \ + +#define XR_LIST_STRUCT_XrSceneObjectsMSFT(_) \ + _(type) \ + _(next) \ + _(sceneObjectCount) \ + _(sceneObjects) \ + +#define XR_LIST_STRUCT_XrSceneComponentParentFilterInfoMSFT(_) \ + _(type) \ + _(next) \ + _(parentId) \ + +#define XR_LIST_STRUCT_XrSceneObjectTypesFilterInfoMSFT(_) \ + _(type) \ + _(next) \ + _(objectTypeCount) \ + _(objectTypes) \ + +#define XR_LIST_STRUCT_XrScenePlaneMSFT(_) \ + _(alignment) \ + _(size) \ + _(meshBufferId) \ + _(supportsIndicesUint16) \ + +#define XR_LIST_STRUCT_XrScenePlanesMSFT(_) \ + _(type) \ + _(next) \ + _(scenePlaneCount) \ + _(scenePlanes) \ + +#define XR_LIST_STRUCT_XrScenePlaneAlignmentFilterInfoMSFT(_) \ + _(type) \ + _(next) \ + _(alignmentCount) \ + _(alignments) \ + +#define XR_LIST_STRUCT_XrSceneMeshMSFT(_) \ + _(meshBufferId) \ + _(supportsIndicesUint16) \ + +#define XR_LIST_STRUCT_XrSceneMeshesMSFT(_) \ + _(type) \ + _(next) \ + _(sceneMeshCount) \ + _(sceneMeshes) \ + +#define XR_LIST_STRUCT_XrSceneMeshBuffersGetInfoMSFT(_) \ + _(type) \ + _(next) \ + _(meshBufferId) \ + +#define XR_LIST_STRUCT_XrSceneMeshBuffersMSFT(_) \ + _(type) \ + _(next) \ + +#define XR_LIST_STRUCT_XrSceneMeshVertexBufferMSFT(_) \ + _(type) \ + _(next) \ + _(vertexCapacityInput) \ + _(vertexCountOutput) \ + _(vertices) \ + +#define XR_LIST_STRUCT_XrSceneMeshIndicesUint32MSFT(_) \ + _(type) \ + _(next) \ + _(indexCapacityInput) \ + _(indexCountOutput) \ + _(indices) \ + +#define XR_LIST_STRUCT_XrSceneMeshIndicesUint16MSFT(_) \ + _(type) \ + _(next) \ + _(indexCapacityInput) \ + _(indexCountOutput) \ + _(indices) \ + +#define XR_LIST_STRUCT_XrSerializedSceneFragmentDataGetInfoMSFT(_) \ + _(type) \ + _(next) \ + _(sceneFragmentId) \ + +#define XR_LIST_STRUCT_XrDeserializeSceneFragmentMSFT(_) \ + _(bufferSize) \ + _(buffer) \ + +#define XR_LIST_STRUCT_XrSceneDeserializeInfoMSFT(_) \ + _(type) \ + _(next) \ + _(fragmentCount) \ + _(fragments) \ + #define XR_LIST_STRUCT_XrEventDataDisplayRefreshRateChangedFB(_) \ _(type) \ _(next) \ @@ -1420,6 +1819,177 @@ XR_ENUM_STR(XrResult); _(next) \ _(colorSpace) \ +#define XR_LIST_STRUCT_XrVector4sFB(_) \ + _(x) \ + _(y) \ + _(z) \ + _(w) \ + +#define XR_LIST_STRUCT_XrHandTrackingMeshFB(_) \ + _(type) \ + _(next) \ + _(jointCapacityInput) \ + _(jointCountOutput) \ + _(jointBindPoses) \ + _(jointRadii) \ + _(jointParents) \ + _(vertexCapacityInput) \ + _(vertexCountOutput) \ + _(vertexPositions) \ + _(vertexNormals) \ + _(vertexUVs) \ + _(vertexBlendIndices) \ + _(vertexBlendWeights) \ + _(indexCapacityInput) \ + _(indexCountOutput) \ + _(indices) \ + +#define XR_LIST_STRUCT_XrHandTrackingScaleFB(_) \ + _(type) \ + _(next) \ + _(sensorOutput) \ + _(currentOutput) \ + _(overrideHandScale) \ + _(overrideValueInput) \ + +#define XR_LIST_STRUCT_XrHandTrackingAimStateFB(_) \ + _(type) \ + _(next) \ + _(status) \ + _(aimPose) \ + _(pinchStrengthIndex) \ + _(pinchStrengthMiddle) \ + _(pinchStrengthRing) \ + _(pinchStrengthLittle) \ + +#define XR_LIST_STRUCT_XrHandCapsuleFB(_) \ + _(points) \ + _(radius) \ + _(joint) \ + +#define XR_LIST_STRUCT_XrHandTrackingCapsulesStateFB(_) \ + _(type) \ + _(next) \ + _(capsules) \ + +#define XR_LIST_STRUCT_XrFoveationProfileCreateInfoFB(_) \ + _(type) \ + _(next) \ + +#define XR_LIST_STRUCT_XrSwapchainCreateInfoFoveationFB(_) \ + _(type) \ + _(next) \ + _(flags) \ + +#define XR_LIST_STRUCT_XrSwapchainStateFoveationFB(_) \ + _(type) \ + _(next) \ + _(flags) \ + _(profile) \ + +#define XR_LIST_STRUCT_XrFoveationLevelProfileCreateInfoFB(_) \ + _(type) \ + _(next) \ + _(level) \ + _(verticalOffset) \ + _(dynamic) \ + +#define XR_LIST_STRUCT_XrViewLocateFoveatedRenderingVARJO(_) \ + _(type) \ + _(next) \ + _(foveatedRenderingActive) \ + +#define XR_LIST_STRUCT_XrFoveatedViewConfigurationViewVARJO(_) \ + _(type) \ + _(next) \ + _(foveatedRenderingActive) \ + +#define XR_LIST_STRUCT_XrSystemFoveatedRenderingPropertiesVARJO(_) \ + _(type) \ + _(next) \ + _(supportsFoveatedRendering) \ + +#define XR_LIST_STRUCT_XrCompositionLayerDepthTestVARJO(_) \ + _(type) \ + _(next) \ + _(depthTestRangeNearZ) \ + _(depthTestRangeFarZ) \ + +#define XR_LIST_STRUCT_XrSpatialAnchorPersistenceNameMSFT(_) \ + _(name) \ + +#define XR_LIST_STRUCT_XrSpatialAnchorPersistenceInfoMSFT(_) \ + _(type) \ + _(next) \ + _(spatialAnchorPersistenceName) \ + _(spatialAnchor) \ + +#define XR_LIST_STRUCT_XrSpatialAnchorFromPersistedAnchorCreateInfoMSFT(_) \ + _(type) \ + _(next) \ + _(spatialAnchorStore) \ + _(spatialAnchorPersistenceName) \ + +#define XR_LIST_STRUCT_XrSwapchainImageFoveationVulkanFB(_) \ + _(type) \ + _(next) \ + _(image) \ + _(width) \ + _(height) \ + +#define XR_LIST_STRUCT_XrSwapchainStateAndroidSurfaceDimensionsFB(_) \ + _(type) \ + _(next) \ + _(width) \ + _(height) \ + +#define XR_LIST_STRUCT_XrSwapchainStateSamplerOpenGLESFB(_) \ + _(type) \ + _(next) \ + _(minFilter) \ + _(magFilter) \ + _(wrapModeS) \ + _(wrapModeT) \ + _(swizzleRed) \ + _(swizzleGreen) \ + _(swizzleBlue) \ + _(swizzleAlpha) \ + _(maxAnisotropy) \ + _(borderColor) \ + +#define XR_LIST_STRUCT_XrSwapchainStateSamplerVulkanFB(_) \ + _(type) \ + _(next) \ + _(minFilter) \ + _(magFilter) \ + _(mipmapMode) \ + _(wrapModeS) \ + _(wrapModeT) \ + _(swizzleRed) \ + _(swizzleGreen) \ + _(swizzleBlue) \ + _(swizzleAlpha) \ + _(maxAnisotropy) \ + _(borderColor) \ + +#define XR_LIST_STRUCT_XrCompositionLayerSpaceWarpInfoFB(_) \ + _(type) \ + _(next) \ + _(layerFlags) \ + _(motionVectorSubImage) \ + _(appSpaceDeltaPose) \ + _(depthSubImage) \ + _(minDepth) \ + _(maxDepth) \ + _(nearZ) \ + _(farZ) \ + +#define XR_LIST_STRUCT_XrSystemSpaceWarpPropertiesFB(_) \ + _(type) \ + _(next) \ + _(recommendedMotionVectorImageRectWidth) \ + _(recommendedMotionVectorImageRectHeight) \ + #define XR_LIST_STRUCTURE_TYPES_CORE(_) \ @@ -1492,6 +2062,8 @@ XR_ENUM_STR(XrResult); _(XrEventDataMainSessionVisibilityChangedEXTX, XR_TYPE_EVENT_DATA_MAIN_SESSION_VISIBILITY_CHANGED_EXTX) \ _(XrSpatialAnchorCreateInfoMSFT, XR_TYPE_SPATIAL_ANCHOR_CREATE_INFO_MSFT) \ _(XrSpatialAnchorSpaceCreateInfoMSFT, XR_TYPE_SPATIAL_ANCHOR_SPACE_CREATE_INFO_MSFT) \ + _(XrCompositionLayerImageLayoutFB, XR_TYPE_COMPOSITION_LAYER_IMAGE_LAYOUT_FB) \ + _(XrCompositionLayerAlphaBlendFB, XR_TYPE_COMPOSITION_LAYER_ALPHA_BLEND_FB) \ _(XrViewConfigurationDepthRangeEXT, XR_TYPE_VIEW_CONFIGURATION_DEPTH_RANGE_EXT) \ _(XrSpatialGraphNodeSpaceCreateInfoMSFT, XR_TYPE_SPATIAL_GRAPH_NODE_SPACE_CREATE_INFO_MSFT) \ _(XrSystemHandTrackingPropertiesEXT, XR_TYPE_SYSTEM_HAND_TRACKING_PROPERTIES_EXT) \ @@ -1516,9 +2088,50 @@ XR_ENUM_STR(XrResult); _(XrControllerModelNodeStateMSFT, XR_TYPE_CONTROLLER_MODEL_NODE_STATE_MSFT) \ _(XrControllerModelStateMSFT, XR_TYPE_CONTROLLER_MODEL_STATE_MSFT) \ _(XrViewConfigurationViewFovEPIC, XR_TYPE_VIEW_CONFIGURATION_VIEW_FOV_EPIC) \ + _(XrCompositionLayerReprojectionInfoMSFT, XR_TYPE_COMPOSITION_LAYER_REPROJECTION_INFO_MSFT) \ + _(XrCompositionLayerReprojectionPlaneOverrideMSFT, XR_TYPE_COMPOSITION_LAYER_REPROJECTION_PLANE_OVERRIDE_MSFT) \ + _(XrCompositionLayerSecureContentFB, XR_TYPE_COMPOSITION_LAYER_SECURE_CONTENT_FB) \ _(XrInteractionProfileAnalogThresholdVALVE, XR_TYPE_INTERACTION_PROFILE_ANALOG_THRESHOLD_VALVE) \ + _(XrHandJointsMotionRangeInfoEXT, XR_TYPE_HAND_JOINTS_MOTION_RANGE_INFO_EXT) \ + _(XrSceneObserverCreateInfoMSFT, XR_TYPE_SCENE_OBSERVER_CREATE_INFO_MSFT) \ + _(XrSceneCreateInfoMSFT, XR_TYPE_SCENE_CREATE_INFO_MSFT) \ + _(XrNewSceneComputeInfoMSFT, XR_TYPE_NEW_SCENE_COMPUTE_INFO_MSFT) \ + _(XrVisualMeshComputeLodInfoMSFT, XR_TYPE_VISUAL_MESH_COMPUTE_LOD_INFO_MSFT) \ + _(XrSceneComponentsMSFT, XR_TYPE_SCENE_COMPONENTS_MSFT) \ + _(XrSceneComponentsGetInfoMSFT, XR_TYPE_SCENE_COMPONENTS_GET_INFO_MSFT) \ + _(XrSceneComponentLocationsMSFT, XR_TYPE_SCENE_COMPONENT_LOCATIONS_MSFT) \ + _(XrSceneComponentsLocateInfoMSFT, XR_TYPE_SCENE_COMPONENTS_LOCATE_INFO_MSFT) \ + _(XrSceneObjectsMSFT, XR_TYPE_SCENE_OBJECTS_MSFT) \ + _(XrSceneComponentParentFilterInfoMSFT, XR_TYPE_SCENE_COMPONENT_PARENT_FILTER_INFO_MSFT) \ + _(XrSceneObjectTypesFilterInfoMSFT, XR_TYPE_SCENE_OBJECT_TYPES_FILTER_INFO_MSFT) \ + _(XrScenePlanesMSFT, XR_TYPE_SCENE_PLANES_MSFT) \ + _(XrScenePlaneAlignmentFilterInfoMSFT, XR_TYPE_SCENE_PLANE_ALIGNMENT_FILTER_INFO_MSFT) \ + _(XrSceneMeshesMSFT, XR_TYPE_SCENE_MESHES_MSFT) \ + _(XrSceneMeshBuffersGetInfoMSFT, XR_TYPE_SCENE_MESH_BUFFERS_GET_INFO_MSFT) \ + _(XrSceneMeshBuffersMSFT, XR_TYPE_SCENE_MESH_BUFFERS_MSFT) \ + _(XrSceneMeshVertexBufferMSFT, XR_TYPE_SCENE_MESH_VERTEX_BUFFER_MSFT) \ + _(XrSceneMeshIndicesUint32MSFT, XR_TYPE_SCENE_MESH_INDICES_UINT32_MSFT) \ + _(XrSceneMeshIndicesUint16MSFT, XR_TYPE_SCENE_MESH_INDICES_UINT16_MSFT) \ + _(XrSerializedSceneFragmentDataGetInfoMSFT, XR_TYPE_SERIALIZED_SCENE_FRAGMENT_DATA_GET_INFO_MSFT) \ + _(XrSceneDeserializeInfoMSFT, XR_TYPE_SCENE_DESERIALIZE_INFO_MSFT) \ _(XrEventDataDisplayRefreshRateChangedFB, XR_TYPE_EVENT_DATA_DISPLAY_REFRESH_RATE_CHANGED_FB) \ _(XrSystemColorSpacePropertiesFB, XR_TYPE_SYSTEM_COLOR_SPACE_PROPERTIES_FB) \ + _(XrHandTrackingMeshFB, XR_TYPE_HAND_TRACKING_MESH_FB) \ + _(XrHandTrackingScaleFB, XR_TYPE_HAND_TRACKING_SCALE_FB) \ + _(XrHandTrackingAimStateFB, XR_TYPE_HAND_TRACKING_AIM_STATE_FB) \ + _(XrHandTrackingCapsulesStateFB, XR_TYPE_HAND_TRACKING_CAPSULES_STATE_FB) \ + _(XrFoveationProfileCreateInfoFB, XR_TYPE_FOVEATION_PROFILE_CREATE_INFO_FB) \ + _(XrSwapchainCreateInfoFoveationFB, XR_TYPE_SWAPCHAIN_CREATE_INFO_FOVEATION_FB) \ + _(XrSwapchainStateFoveationFB, XR_TYPE_SWAPCHAIN_STATE_FOVEATION_FB) \ + _(XrFoveationLevelProfileCreateInfoFB, XR_TYPE_FOVEATION_LEVEL_PROFILE_CREATE_INFO_FB) \ + _(XrViewLocateFoveatedRenderingVARJO, XR_TYPE_VIEW_LOCATE_FOVEATED_RENDERING_VARJO) \ + _(XrFoveatedViewConfigurationViewVARJO, XR_TYPE_FOVEATED_VIEW_CONFIGURATION_VIEW_VARJO) \ + _(XrSystemFoveatedRenderingPropertiesVARJO, XR_TYPE_SYSTEM_FOVEATED_RENDERING_PROPERTIES_VARJO) \ + _(XrCompositionLayerDepthTestVARJO, XR_TYPE_COMPOSITION_LAYER_DEPTH_TEST_VARJO) \ + _(XrSpatialAnchorPersistenceInfoMSFT, XR_TYPE_SPATIAL_ANCHOR_PERSISTENCE_INFO_MSFT) \ + _(XrSpatialAnchorFromPersistedAnchorCreateInfoMSFT, XR_TYPE_SPATIAL_ANCHOR_FROM_PERSISTED_ANCHOR_CREATE_INFO_MSFT) \ + _(XrCompositionLayerSpaceWarpInfoFB, XR_TYPE_COMPOSITION_LAYER_SPACE_WARP_INFO_FB) \ + _(XrSystemSpaceWarpPropertiesFB, XR_TYPE_SYSTEM_SPACE_WARP_PROPERTIES_FB) \ @@ -1595,6 +2208,7 @@ XR_ENUM_STR(XrResult); #define XR_LIST_STRUCTURE_TYPES_XR_USE_GRAPHICS_API_OPENGL_ES(_) \ _(XrSwapchainImageOpenGLESKHR, XR_TYPE_SWAPCHAIN_IMAGE_OPENGL_ES_KHR) \ _(XrGraphicsRequirementsOpenGLESKHR, XR_TYPE_GRAPHICS_REQUIREMENTS_OPENGL_ES_KHR) \ + _(XrSwapchainStateSamplerOpenGLESFB, XR_TYPE_SWAPCHAIN_STATE_SAMPLER_OPENGL_ES_FB) \ #else @@ -1619,6 +2233,8 @@ XR_ENUM_STR(XrResult); _(XrVulkanInstanceCreateInfoKHR, XR_TYPE_VULKAN_INSTANCE_CREATE_INFO_KHR) \ _(XrVulkanDeviceCreateInfoKHR, XR_TYPE_VULKAN_DEVICE_CREATE_INFO_KHR) \ _(XrVulkanGraphicsDeviceGetInfoKHR, XR_TYPE_VULKAN_GRAPHICS_DEVICE_GET_INFO_KHR) \ + _(XrSwapchainImageFoveationVulkanFB, XR_TYPE_SWAPCHAIN_IMAGE_FOVEATION_VULKAN_FB) \ + _(XrSwapchainStateSamplerVulkanFB, XR_TYPE_SWAPCHAIN_STATE_SAMPLER_VULKAN_FB) \ #else @@ -1629,6 +2245,8 @@ XR_ENUM_STR(XrResult); #define XR_LIST_STRUCTURE_TYPES_XR_USE_PLATFORM_ANDROID(_) \ _(XrInstanceCreateInfoAndroidKHR, XR_TYPE_INSTANCE_CREATE_INFO_ANDROID_KHR) \ _(XrLoaderInitInfoAndroidKHR, XR_TYPE_LOADER_INIT_INFO_ANDROID_KHR) \ + _(XrAndroidSurfaceSwapchainCreateInfoFB, XR_TYPE_ANDROID_SURFACE_SWAPCHAIN_CREATE_INFO_FB) \ + _(XrSwapchainStateAndroidSurfaceDimensionsFB, XR_TYPE_SWAPCHAIN_STATE_ANDROID_SURFACE_DIMENSIONS_FB) \ #else @@ -1696,6 +2314,8 @@ XR_ENUM_STR(XrResult); _(XR_VARJO_quad_views, 38) \ _(XR_MSFT_unbounded_reference_space, 39) \ _(XR_MSFT_spatial_anchor, 40) \ + _(XR_FB_composition_layer_image_layout, 41) \ + _(XR_FB_composition_layer_alpha_blend, 42) \ _(XR_MND_headless, 43) \ _(XR_OCULUS_android_session_state_enable, 45) \ _(XR_EXT_view_configuration_depth_range, 47) \ @@ -1712,8 +2332,13 @@ XR_ENUM_STR(XrResult); _(XR_EXT_win32_appcontainer_compatible, 58) \ _(XR_EPIC_view_configuration_fov, 60) \ _(XR_MSFT_holographic_window_attachment, 64) \ + _(XR_MSFT_composition_layer_reprojection, 67) \ _(XR_HUAWEI_controller_interaction, 70) \ + _(XR_FB_android_surface_swapchain_create, 71) \ + _(XR_FB_swapchain_update_state, 72) \ + _(XR_FB_composition_layer_secure_content, 73) \ _(XR_VALVE_analog_threshold, 80) \ + _(XR_EXT_hand_joints_motion_range, 81) \ _(XR_KHR_loader_init, 89) \ _(XR_KHR_loader_init_android, 90) \ _(XR_KHR_vulkan_enable2, 91) \ @@ -1721,10 +2346,28 @@ XR_ENUM_STR(XrResult); _(XR_EXT_samsung_odyssey_controller, 95) \ _(XR_EXT_hp_mixed_reality_controller, 96) \ _(XR_MND_swapchain_usage_input_attachment_bit, 97) \ + _(XR_MSFT_scene_understanding, 98) \ + _(XR_MSFT_scene_understanding_serialization, 99) \ _(XR_FB_display_refresh_rate, 102) \ _(XR_HTC_vive_cosmos_controller_interaction, 103) \ _(XR_FB_color_space, 109) \ + _(XR_FB_hand_tracking_mesh, 111) \ + _(XR_FB_hand_tracking_aim, 112) \ + _(XR_FB_hand_tracking_capsules, 113) \ + _(XR_FB_foveation, 115) \ + _(XR_FB_foveation_configuration, 116) \ _(XR_KHR_binding_modification, 121) \ + _(XR_VARJO_foveated_rendering, 122) \ + _(XR_VARJO_composition_layer_depth_test, 123) \ + _(XR_VARJO_environment_depth_estimation, 124) \ + _(XR_MSFT_spatial_anchor_persistence, 143) \ + _(XR_OCULUS_audio_device_guid, 160) \ + _(XR_FB_foveation_vulkan, 161) \ + _(XR_FB_swapchain_update_state_android_surface, 162) \ + _(XR_FB_swapchain_update_state_opengl_es, 163) \ + _(XR_FB_swapchain_update_state_vulkan, 164) \ + _(XR_KHR_swapchain_usage_input_attachment_bit, 166) \ + _(XR_FB_space_warp, 172) \ #endif diff --git a/src/external/slam_tracker/slam_tracker.hpp b/src/external/slam_tracker/slam_tracker.hpp new file mode 100644 index 000000000..cd39c7d35 --- /dev/null +++ b/src/external/slam_tracker/slam_tracker.hpp @@ -0,0 +1,111 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief SLAM tracker class header for usage in Monado. + * @author Mateo de Mayo + * @ingroup aux_tracking + * + * This header file contains the declaration of the @ref slam_tracker class to + * implement a SLAM system for usage in Monado. + * + * A copy of this file is present in both Monado and any SLAM system that + * intends to be used by Monado. The SLAM system should provide the + * `slam_tracker` implementation so that Monado can use it. + * + * This file also declares additional data types to be used between Monado and + * the system. + * + * @todo The interface is preliminary and should be improved to avoid + * unnecessary copies. + */ + +#pragma once + +#include + +namespace xrt::auxiliary::tracking::slam { + +/*! + * @brief Standard pose type to communicate Monado with the external SLAM system + */ +struct pose { + float px, py, pz; + float rx, ry, rz, rw; + pose() = default; + pose(float px, float py, float pz, float rx, float ry, float rz, float rw) + : px(px), py(py), pz(pz), rx(rx), ry(ry), rz(rz), rw(rw) {} +}; + +/*! + * @brief IMU Sample type to pass around between programs + */ +struct imu_sample { + std::int64_t timestamp; // In nanoseconds + double ax, ay, az; // In meters per second squared (m / s^2) + double wx, wy, wz; // In radians per second (rad / s) + imu_sample() = default; + imu_sample(std::int64_t timestamp, double ax, double ay, double az, double wx, + double wy, double wz) + : timestamp(timestamp), ax(ax), ay(ay), az(az), wx(wx), wy(wy), wz(wz) {} +}; + +/*! + * @brief Image sample type to pass around between programs. It is expected that + * any SLAM system takes OpenCV matrices as input. + */ +struct img_sample { + std::int64_t timestamp; + cv::Mat img; + bool is_left; + img_sample() = default; + img_sample(std::int64_t timestamp, cv::Mat img, bool is_left) + : timestamp(timestamp), img(img), is_left(is_left) {} +}; + +/*! + * @brief slam_tracker serves as an interface between Monado and external SLAM + * systems. + * + * This class uses the pointer-to-implementation pattern, and its implementation + * should be provided by an external SLAM system. + */ +struct slam_tracker { + /*! + * @brief Construct a new slam tracker object + * + * @param config_file SLAM systems parameters tend to be numerous and very + * specific, so they usually use a configuration file as the main way to set + * them up. Therefore, this constructor receives a path to a + * implementation-specific configuration file. + */ + slam_tracker(std::string config_file); + ~slam_tracker(); + + slam_tracker(const slam_tracker &) = delete; + slam_tracker &operator=(const slam_tracker &) = delete; + + void start(); + void stop(); + bool is_running(); + + //! There must be a single producer thread pushing samples. + //! Samples must have monotonically increasing timestamps. + //! The implementation must be non-blocking. + //! A separate consumer thread should process the samples. + void push_imu_sample(imu_sample sample); + + //! Same conditions as `push_imu_sample` apply. + //! When using stereo frames, they must be pushed in a left-right order. + //! The consecutive left-right pair must have the same timestamps. + void push_frame(img_sample sample); + + //! There must be a single thread accessing the tracked pose. + bool try_dequeue_pose(pose &pose); + + private: + struct implementation; + implementation *impl; +}; + +} // namespace xrt::auxiliary::tracking::slam diff --git a/src/external/stb/stb_image_write.h b/src/external/stb/stb_image_write.h new file mode 100644 index 000000000..cffd473c1 --- /dev/null +++ b/src/external/stb/stb_image_write.h @@ -0,0 +1,1666 @@ +/* stb_image_write - v1.14 - public domain - http://nothings.org/stb + writes out PNG/BMP/TGA/JPEG/HDR images to C stdio - Sean Barrett 2010-2015 + no warranty implied; use at your own risk + + Before #including, + + #define STB_IMAGE_WRITE_IMPLEMENTATION + + in the file that you want to have the implementation. + + Will probably not work correctly with strict-aliasing optimizations. + +ABOUT: + + This header file is a library for writing images to C stdio or a callback. + + The PNG output is not optimal; it is 20-50% larger than the file + written by a decent optimizing implementation; though providing a custom + zlib compress function (see STBIW_ZLIB_COMPRESS) can mitigate that. + This library is designed for source code compactness and simplicity, + not optimal image file size or run-time performance. + +BUILDING: + + You can #define STBIW_ASSERT(x) before the #include to avoid using assert.h. + You can #define STBIW_MALLOC(), STBIW_REALLOC(), and STBIW_FREE() to replace + malloc,realloc,free. + You can #define STBIW_MEMMOVE() to replace memmove() + You can #define STBIW_ZLIB_COMPRESS to use a custom zlib-style compress function + for PNG compression (instead of the builtin one), it must have the following signature: + unsigned char * my_compress(unsigned char *data, int data_len, int *out_len, int quality); + The returned data will be freed with STBIW_FREE() (free() by default), + so it must be heap allocated with STBIW_MALLOC() (malloc() by default), + +UNICODE: + + If compiling for Windows and you wish to use Unicode filenames, compile + with + #define STBIW_WINDOWS_UTF8 + and pass utf8-encoded filenames. Call stbiw_convert_wchar_to_utf8 to convert + Windows wchar_t filenames to utf8. + +USAGE: + + There are five functions, one for each image file format: + + int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); + int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); + int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); + int stbi_write_jpg(char const *filename, int w, int h, int comp, const void *data, int quality); + int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); + + void stbi_flip_vertically_on_write(int flag); // flag is non-zero to flip data vertically + + There are also five equivalent functions that use an arbitrary write function. You are + expected to open/close your file-equivalent before and after calling these: + + int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); + int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); + int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); + int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); + int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); + + where the callback is: + void stbi_write_func(void *context, void *data, int size); + + You can configure it with these global variables: + int stbi_write_tga_with_rle; // defaults to true; set to 0 to disable RLE + int stbi_write_png_compression_level; // defaults to 8; set to higher for more compression + int stbi_write_force_png_filter; // defaults to -1; set to 0..5 to force a filter mode + + + You can define STBI_WRITE_NO_STDIO to disable the file variant of these + functions, so the library will not use stdio.h at all. However, this will + also disable HDR writing, because it requires stdio for formatted output. + + Each function returns 0 on failure and non-0 on success. + + The functions create an image file defined by the parameters. The image + is a rectangle of pixels stored from left-to-right, top-to-bottom. + Each pixel contains 'comp' channels of data stored interleaved with 8-bits + per channel, in the following order: 1=Y, 2=YA, 3=RGB, 4=RGBA. (Y is + monochrome color.) The rectangle is 'w' pixels wide and 'h' pixels tall. + The *data pointer points to the first byte of the top-left-most pixel. + For PNG, "stride_in_bytes" is the distance in bytes from the first byte of + a row of pixels to the first byte of the next row of pixels. + + PNG creates output files with the same number of components as the input. + The BMP format expands Y to RGB in the file format and does not + output alpha. + + PNG supports writing rectangles of data even when the bytes storing rows of + data are not consecutive in memory (e.g. sub-rectangles of a larger image), + by supplying the stride between the beginning of adjacent rows. The other + formats do not. (Thus you cannot write a native-format BMP through the BMP + writer, both because it is in BGR order and because it may have padding + at the end of the line.) + + PNG allows you to set the deflate compression level by setting the global + variable 'stbi_write_png_compression_level' (it defaults to 8). + + HDR expects linear float data. Since the format is always 32-bit rgb(e) + data, alpha (if provided) is discarded, and for monochrome data it is + replicated across all three channels. + + TGA supports RLE or non-RLE compressed data. To use non-RLE-compressed + data, set the global variable 'stbi_write_tga_with_rle' to 0. + + JPEG does ignore alpha channels in input data; quality is between 1 and 100. + Higher quality looks better but results in a bigger image. + JPEG baseline (no JPEG progressive). + +CREDITS: + + + Sean Barrett - PNG/BMP/TGA + Baldur Karlsson - HDR + Jean-Sebastien Guay - TGA monochrome + Tim Kelsey - misc enhancements + Alan Hickman - TGA RLE + Emmanuel Julien - initial file IO callback implementation + Jon Olick - original jo_jpeg.cpp code + Daniel Gibson - integrate JPEG, allow external zlib + Aarni Koskela - allow choosing PNG filter + + bugfixes: + github:Chribba + Guillaume Chereau + github:jry2 + github:romigrou + Sergio Gonzalez + Jonas Karlsson + Filip Wasil + Thatcher Ulrich + github:poppolopoppo + Patrick Boettcher + github:xeekworx + Cap Petschulat + Simon Rodriguez + Ivan Tikhonov + github:ignotion + Adam Schackart + +LICENSE + + See end of file for license information. + +*/ + +#ifndef INCLUDE_STB_IMAGE_WRITE_H +#define INCLUDE_STB_IMAGE_WRITE_H + +#include + +// if STB_IMAGE_WRITE_STATIC causes problems, try defining STBIWDEF to 'inline' or 'static inline' +#ifndef STBIWDEF +#ifdef STB_IMAGE_WRITE_STATIC +#define STBIWDEF static +#else +#ifdef __cplusplus +#define STBIWDEF extern "C" +#else +#define STBIWDEF extern +#endif +#endif +#endif + +#ifndef STB_IMAGE_WRITE_STATIC // C++ forbids static forward declarations +extern int stbi_write_tga_with_rle; +extern int stbi_write_png_compression_level; +extern int stbi_write_force_png_filter; +#endif + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality); + +#ifdef STBI_WINDOWS_UTF8 +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); +#endif +#endif + +typedef void stbi_write_func(void *context, void *data, int size); + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); + +STBIWDEF void stbi_flip_vertically_on_write(int flip_boolean); + +#endif//INCLUDE_STB_IMAGE_WRITE_H + +#ifdef STB_IMAGE_WRITE_IMPLEMENTATION + +#ifdef _WIN32 + #ifndef _CRT_SECURE_NO_WARNINGS + #define _CRT_SECURE_NO_WARNINGS + #endif + #ifndef _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_NONSTDC_NO_DEPRECATE + #endif +#endif + +#ifndef STBI_WRITE_NO_STDIO +#include +#endif // STBI_WRITE_NO_STDIO + +#include +#include +#include +#include + +#if defined(STBIW_MALLOC) && defined(STBIW_FREE) && (defined(STBIW_REALLOC) || defined(STBIW_REALLOC_SIZED)) +// ok +#elif !defined(STBIW_MALLOC) && !defined(STBIW_FREE) && !defined(STBIW_REALLOC) && !defined(STBIW_REALLOC_SIZED) +// ok +#else +#error "Must define all or none of STBIW_MALLOC, STBIW_FREE, and STBIW_REALLOC (or STBIW_REALLOC_SIZED)." +#endif + +#ifndef STBIW_MALLOC +#define STBIW_MALLOC(sz) malloc(sz) +#define STBIW_REALLOC(p,newsz) realloc(p,newsz) +#define STBIW_FREE(p) free(p) +#endif + +#ifndef STBIW_REALLOC_SIZED +#define STBIW_REALLOC_SIZED(p,oldsz,newsz) STBIW_REALLOC(p,newsz) +#endif + + +#ifndef STBIW_MEMMOVE +#define STBIW_MEMMOVE(a,b,sz) memmove(a,b,sz) +#endif + + +#ifndef STBIW_ASSERT +#include +#define STBIW_ASSERT(x) assert(x) +#endif + +#define STBIW_UCHAR(x) (unsigned char) ((x) & 0xff) + +#ifdef STB_IMAGE_WRITE_STATIC +static int stbi_write_png_compression_level = 8; +static int stbi_write_tga_with_rle = 1; +static int stbi_write_force_png_filter = -1; +#else +int stbi_write_png_compression_level = 8; +int stbi_write_tga_with_rle = 1; +int stbi_write_force_png_filter = -1; +#endif + +static int stbi__flip_vertically_on_write = 0; + +STBIWDEF void stbi_flip_vertically_on_write(int flag) +{ + stbi__flip_vertically_on_write = flag; +} + +typedef struct +{ + stbi_write_func *func; + void *context; +} stbi__write_context; + +// initialize a callback-based context +static void stbi__start_write_callbacks(stbi__write_context *s, stbi_write_func *c, void *context) +{ + s->func = c; + s->context = context; +} + +#ifndef STBI_WRITE_NO_STDIO + +static void stbi__stdio_write(void *context, void *data, int size) +{ + fwrite(data,1,size,(FILE*) context); +} + +#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) +#ifdef __cplusplus +#define STBIW_EXTERN extern "C" +#else +#define STBIW_EXTERN extern +#endif +STBIW_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); +STBIW_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); + +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) +{ + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL); +} +#endif + +static FILE *stbiw__fopen(char const *filename, char const *mode) +{ + FILE *f; +#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode))) + return 0; + +#if _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f=0; +#else + f = fopen(filename, mode); +#endif + return f; +} + +static int stbi__start_write_file(stbi__write_context *s, const char *filename) +{ + FILE *f = stbiw__fopen(filename, "wb"); + stbi__start_write_callbacks(s, stbi__stdio_write, (void *) f); + return f != NULL; +} + +static void stbi__end_write_file(stbi__write_context *s) +{ + fclose((FILE *)s->context); +} + +#endif // !STBI_WRITE_NO_STDIO + +typedef unsigned int stbiw_uint32; +typedef int stb_image_write_test[sizeof(stbiw_uint32)==4 ? 1 : -1]; + +static void stbiw__writefv(stbi__write_context *s, const char *fmt, va_list v) +{ + while (*fmt) { + switch (*fmt++) { + case ' ': break; + case '1': { unsigned char x = STBIW_UCHAR(va_arg(v, int)); + s->func(s->context,&x,1); + break; } + case '2': { int x = va_arg(v,int); + unsigned char b[2]; + b[0] = STBIW_UCHAR(x); + b[1] = STBIW_UCHAR(x>>8); + s->func(s->context,b,2); + break; } + case '4': { stbiw_uint32 x = va_arg(v,int); + unsigned char b[4]; + b[0]=STBIW_UCHAR(x); + b[1]=STBIW_UCHAR(x>>8); + b[2]=STBIW_UCHAR(x>>16); + b[3]=STBIW_UCHAR(x>>24); + s->func(s->context,b,4); + break; } + default: + STBIW_ASSERT(0); + return; + } + } +} + +static void stbiw__writef(stbi__write_context *s, const char *fmt, ...) +{ + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); +} + +static void stbiw__putc(stbi__write_context *s, unsigned char c) +{ + s->func(s->context, &c, 1); +} + +static void stbiw__write3(stbi__write_context *s, unsigned char a, unsigned char b, unsigned char c) +{ + unsigned char arr[3]; + arr[0] = a; arr[1] = b; arr[2] = c; + s->func(s->context, arr, 3); +} + +static void stbiw__write_pixel(stbi__write_context *s, int rgb_dir, int comp, int write_alpha, int expand_mono, unsigned char *d) +{ + unsigned char bg[3] = { 255, 0, 255}, px[3]; + int k; + + if (write_alpha < 0) + s->func(s->context, &d[comp - 1], 1); + + switch (comp) { + case 2: // 2 pixels = mono + alpha, alpha is written separately, so same as 1-channel case + case 1: + if (expand_mono) + stbiw__write3(s, d[0], d[0], d[0]); // monochrome bmp + else + s->func(s->context, d, 1); // monochrome TGA + break; + case 4: + if (!write_alpha) { + // composite against pink background + for (k = 0; k < 3; ++k) + px[k] = bg[k] + ((d[k] - bg[k]) * d[3]) / 255; + stbiw__write3(s, px[1 - rgb_dir], px[1], px[1 + rgb_dir]); + break; + } + /* FALLTHROUGH */ + case 3: + stbiw__write3(s, d[1 - rgb_dir], d[1], d[1 + rgb_dir]); + break; + } + if (write_alpha > 0) + s->func(s->context, &d[comp - 1], 1); +} + +static void stbiw__write_pixels(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, void *data, int write_alpha, int scanline_pad, int expand_mono) +{ + stbiw_uint32 zero = 0; + int i,j, j_end; + + if (y <= 0) + return; + + if (stbi__flip_vertically_on_write) + vdir *= -1; + + if (vdir < 0) { + j_end = -1; j = y-1; + } else { + j_end = y; j = 0; + } + + for (; j != j_end; j += vdir) { + for (i=0; i < x; ++i) { + unsigned char *d = (unsigned char *) data + (j*x+i)*comp; + stbiw__write_pixel(s, rgb_dir, comp, write_alpha, expand_mono, d); + } + s->func(s->context, &zero, scanline_pad); + } +} + +static int stbiw__outfile(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, int expand_mono, void *data, int alpha, int pad, const char *fmt, ...) +{ + if (y < 0 || x < 0) { + return 0; + } else { + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); + stbiw__write_pixels(s,rgb_dir,vdir,x,y,comp,data,alpha,pad, expand_mono); + return 1; + } +} + +static int stbi_write_bmp_core(stbi__write_context *s, int x, int y, int comp, const void *data) +{ + int pad = (-x*3) & 3; + return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *) data,0,pad, + "11 4 22 4" "4 44 22 444444", + 'B', 'M', 14+40+(x*3+pad)*y, 0,0, 14+40, // file header + 40, x,y, 1,24, 0,0,0,0,0,0); // bitmap header +} + +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) +{ + stbi__write_context s; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_bmp_core(&s, x, y, comp, data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_bmp(char const *filename, int x, int y, int comp, const void *data) +{ + stbi__write_context s; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_bmp_core(&s, x, y, comp, data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif //!STBI_WRITE_NO_STDIO + +static int stbi_write_tga_core(stbi__write_context *s, int x, int y, int comp, void *data) +{ + int has_alpha = (comp == 2 || comp == 4); + int colorbytes = has_alpha ? comp-1 : comp; + int format = colorbytes < 2 ? 3 : 2; // 3 color channels (RGB/RGBA) = 2, 1 color channel (Y/YA) = 3 + + if (y < 0 || x < 0) + return 0; + + if (!stbi_write_tga_with_rle) { + return stbiw__outfile(s, -1, -1, x, y, comp, 0, (void *) data, has_alpha, 0, + "111 221 2222 11", 0, 0, format, 0, 0, 0, 0, 0, x, y, (colorbytes + has_alpha) * 8, has_alpha * 8); + } else { + int i,j,k; + int jend, jdir; + + stbiw__writef(s, "111 221 2222 11", 0,0,format+8, 0,0,0, 0,0,x,y, (colorbytes + has_alpha) * 8, has_alpha * 8); + + if (stbi__flip_vertically_on_write) { + j = 0; + jend = y; + jdir = 1; + } else { + j = y-1; + jend = -1; + jdir = -1; + } + for (; j != jend; j += jdir) { + unsigned char *row = (unsigned char *) data + j * x * comp; + int len; + + for (i = 0; i < x; i += len) { + unsigned char *begin = row + i * comp; + int diff = 1; + len = 1; + + if (i < x - 1) { + ++len; + diff = memcmp(begin, row + (i + 1) * comp, comp); + if (diff) { + const unsigned char *prev = begin; + for (k = i + 2; k < x && len < 128; ++k) { + if (memcmp(prev, row + k * comp, comp)) { + prev += comp; + ++len; + } else { + --len; + break; + } + } + } else { + for (k = i + 2; k < x && len < 128; ++k) { + if (!memcmp(begin, row + k * comp, comp)) { + ++len; + } else { + break; + } + } + } + } + + if (diff) { + unsigned char header = STBIW_UCHAR(len - 1); + s->func(s->context, &header, 1); + for (k = 0; k < len; ++k) { + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin + k * comp); + } + } else { + unsigned char header = STBIW_UCHAR(len - 129); + s->func(s->context, &header, 1); + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin); + } + } + } + } + return 1; +} + +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) +{ + stbi__write_context s; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_tga_core(&s, x, y, comp, (void *) data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_tga(char const *filename, int x, int y, int comp, const void *data) +{ + stbi__write_context s; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_tga_core(&s, x, y, comp, (void *) data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR writer +// by Baldur Karlsson + +#define stbiw__max(a, b) ((a) > (b) ? (a) : (b)) + +static void stbiw__linear_to_rgbe(unsigned char *rgbe, float *linear) +{ + int exponent; + float maxcomp = stbiw__max(linear[0], stbiw__max(linear[1], linear[2])); + + if (maxcomp < 1e-32f) { + rgbe[0] = rgbe[1] = rgbe[2] = rgbe[3] = 0; + } else { + float normalize = (float) frexp(maxcomp, &exponent) * 256.0f/maxcomp; + + rgbe[0] = (unsigned char)(linear[0] * normalize); + rgbe[1] = (unsigned char)(linear[1] * normalize); + rgbe[2] = (unsigned char)(linear[2] * normalize); + rgbe[3] = (unsigned char)(exponent + 128); + } +} + +static void stbiw__write_run_data(stbi__write_context *s, int length, unsigned char databyte) +{ + unsigned char lengthbyte = STBIW_UCHAR(length+128); + STBIW_ASSERT(length+128 <= 255); + s->func(s->context, &lengthbyte, 1); + s->func(s->context, &databyte, 1); +} + +static void stbiw__write_dump_data(stbi__write_context *s, int length, unsigned char *data) +{ + unsigned char lengthbyte = STBIW_UCHAR(length); + STBIW_ASSERT(length <= 128); // inconsistent with spec but consistent with official code + s->func(s->context, &lengthbyte, 1); + s->func(s->context, data, length); +} + +static void stbiw__write_hdr_scanline(stbi__write_context *s, int width, int ncomp, unsigned char *scratch, float *scanline) +{ + unsigned char scanlineheader[4] = { 2, 2, 0, 0 }; + unsigned char rgbe[4]; + float linear[3]; + int x; + + scanlineheader[2] = (width&0xff00)>>8; + scanlineheader[3] = (width&0x00ff); + + /* skip RLE for images too small or large */ + if (width < 8 || width >= 32768) { + for (x=0; x < width; x++) { + switch (ncomp) { + case 4: /* fallthrough */ + case 3: linear[2] = scanline[x*ncomp + 2]; + linear[1] = scanline[x*ncomp + 1]; + linear[0] = scanline[x*ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + s->func(s->context, rgbe, 4); + } + } else { + int c,r; + /* encode into scratch buffer */ + for (x=0; x < width; x++) { + switch(ncomp) { + case 4: /* fallthrough */ + case 3: linear[2] = scanline[x*ncomp + 2]; + linear[1] = scanline[x*ncomp + 1]; + linear[0] = scanline[x*ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + scratch[x + width*0] = rgbe[0]; + scratch[x + width*1] = rgbe[1]; + scratch[x + width*2] = rgbe[2]; + scratch[x + width*3] = rgbe[3]; + } + + s->func(s->context, scanlineheader, 4); + + /* RLE each component separately */ + for (c=0; c < 4; c++) { + unsigned char *comp = &scratch[width*c]; + + x = 0; + while (x < width) { + // find first run + r = x; + while (r+2 < width) { + if (comp[r] == comp[r+1] && comp[r] == comp[r+2]) + break; + ++r; + } + if (r+2 >= width) + r = width; + // dump up to first run + while (x < r) { + int len = r-x; + if (len > 128) len = 128; + stbiw__write_dump_data(s, len, &comp[x]); + x += len; + } + // if there's a run, output it + if (r+2 < width) { // same test as what we break out of in search loop, so only true if we break'd + // find next byte after run + while (r < width && comp[r] == comp[x]) + ++r; + // output run up to r + while (x < r) { + int len = r-x; + if (len > 127) len = 127; + stbiw__write_run_data(s, len, comp[x]); + x += len; + } + } + } + } + } +} + +static int stbi_write_hdr_core(stbi__write_context *s, int x, int y, int comp, float *data) +{ + if (y <= 0 || x <= 0 || data == NULL) + return 0; + else { + // Each component is stored separately. Allocate scratch space for full output scanline. + unsigned char *scratch = (unsigned char *) STBIW_MALLOC(x*4); + int i, len; + char buffer[128]; + char header[] = "#?RADIANCE\n# Written by stb_image_write.h\nFORMAT=32-bit_rle_rgbe\n"; + s->func(s->context, header, sizeof(header)-1); + +#ifdef __STDC_WANT_SECURE_LIB__ + len = sprintf_s(buffer, sizeof(buffer), "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#else + len = sprintf(buffer, "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#endif + s->func(s->context, buffer, len); + + for(i=0; i < y; i++) + stbiw__write_hdr_scanline(s, x, comp, scratch, data + comp*x*(stbi__flip_vertically_on_write ? y-1-i : i)); + STBIW_FREE(scratch); + return 1; + } +} + +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const float *data) +{ + stbi__write_context s; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_hdr_core(&s, x, y, comp, (float *) data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_hdr(char const *filename, int x, int y, int comp, const float *data) +{ + stbi__write_context s; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_hdr_core(&s, x, y, comp, (float *) data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif // STBI_WRITE_NO_STDIO + + +////////////////////////////////////////////////////////////////////////////// +// +// PNG writer +// + +#ifndef STBIW_ZLIB_COMPRESS +// stretchy buffer; stbiw__sbpush() == vector<>::push_back() -- stbiw__sbcount() == vector<>::size() +#define stbiw__sbraw(a) ((int *) (void *) (a) - 2) +#define stbiw__sbm(a) stbiw__sbraw(a)[0] +#define stbiw__sbn(a) stbiw__sbraw(a)[1] + +#define stbiw__sbneedgrow(a,n) ((a)==0 || stbiw__sbn(a)+n >= stbiw__sbm(a)) +#define stbiw__sbmaybegrow(a,n) (stbiw__sbneedgrow(a,(n)) ? stbiw__sbgrow(a,n) : 0) +#define stbiw__sbgrow(a,n) stbiw__sbgrowf((void **) &(a), (n), sizeof(*(a))) + +#define stbiw__sbpush(a, v) (stbiw__sbmaybegrow(a,1), (a)[stbiw__sbn(a)++] = (v)) +#define stbiw__sbcount(a) ((a) ? stbiw__sbn(a) : 0) +#define stbiw__sbfree(a) ((a) ? STBIW_FREE(stbiw__sbraw(a)),0 : 0) + +static void *stbiw__sbgrowf(void **arr, int increment, int itemsize) +{ + int m = *arr ? 2*stbiw__sbm(*arr)+increment : increment+1; + void *p = STBIW_REALLOC_SIZED(*arr ? stbiw__sbraw(*arr) : 0, *arr ? (stbiw__sbm(*arr)*itemsize + sizeof(int)*2) : 0, itemsize * m + sizeof(int)*2); + STBIW_ASSERT(p); + if (p) { + if (!*arr) ((int *) p)[1] = 0; + *arr = (void *) ((int *) p + 2); + stbiw__sbm(*arr) = m; + } + return *arr; +} + +static unsigned char *stbiw__zlib_flushf(unsigned char *data, unsigned int *bitbuffer, int *bitcount) +{ + while (*bitcount >= 8) { + stbiw__sbpush(data, STBIW_UCHAR(*bitbuffer)); + *bitbuffer >>= 8; + *bitcount -= 8; + } + return data; +} + +static int stbiw__zlib_bitrev(int code, int codebits) +{ + int res=0; + while (codebits--) { + res = (res << 1) | (code & 1); + code >>= 1; + } + return res; +} + +static unsigned int stbiw__zlib_countm(unsigned char *a, unsigned char *b, int limit) +{ + int i; + for (i=0; i < limit && i < 258; ++i) + if (a[i] != b[i]) break; + return i; +} + +static unsigned int stbiw__zhash(unsigned char *data) +{ + stbiw_uint32 hash = data[0] + (data[1] << 8) + (data[2] << 16); + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + return hash; +} + +#define stbiw__zlib_flush() (out = stbiw__zlib_flushf(out, &bitbuf, &bitcount)) +#define stbiw__zlib_add(code,codebits) \ + (bitbuf |= (code) << bitcount, bitcount += (codebits), stbiw__zlib_flush()) +#define stbiw__zlib_huffa(b,c) stbiw__zlib_add(stbiw__zlib_bitrev(b,c),c) +// default huffman tables +#define stbiw__zlib_huff1(n) stbiw__zlib_huffa(0x30 + (n), 8) +#define stbiw__zlib_huff2(n) stbiw__zlib_huffa(0x190 + (n)-144, 9) +#define stbiw__zlib_huff3(n) stbiw__zlib_huffa(0 + (n)-256,7) +#define stbiw__zlib_huff4(n) stbiw__zlib_huffa(0xc0 + (n)-280,8) +#define stbiw__zlib_huff(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : (n) <= 255 ? stbiw__zlib_huff2(n) : (n) <= 279 ? stbiw__zlib_huff3(n) : stbiw__zlib_huff4(n)) +#define stbiw__zlib_huffb(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : stbiw__zlib_huff2(n)) + +#define stbiw__ZHASH 16384 + +#endif // STBIW_ZLIB_COMPRESS + +STBIWDEF unsigned char * stbi_zlib_compress(unsigned char *data, int data_len, int *out_len, int quality) +{ +#ifdef STBIW_ZLIB_COMPRESS + // user provided a zlib compress implementation, use that + return STBIW_ZLIB_COMPRESS(data, data_len, out_len, quality); +#else // use builtin + static unsigned short lengthc[] = { 3,4,5,6,7,8,9,10,11,13,15,17,19,23,27,31,35,43,51,59,67,83,99,115,131,163,195,227,258, 259 }; + static unsigned char lengtheb[]= { 0,0,0,0,0,0,0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 }; + static unsigned short distc[] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577, 32768 }; + static unsigned char disteb[] = { 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13 }; + unsigned int bitbuf=0; + int i,j, bitcount=0; + unsigned char *out = NULL; + unsigned char ***hash_table = (unsigned char***) STBIW_MALLOC(stbiw__ZHASH * sizeof(unsigned char**)); + if (hash_table == NULL) + return NULL; + if (quality < 5) quality = 5; + + stbiw__sbpush(out, 0x78); // DEFLATE 32K window + stbiw__sbpush(out, 0x5e); // FLEVEL = 1 + stbiw__zlib_add(1,1); // BFINAL = 1 + stbiw__zlib_add(1,2); // BTYPE = 1 -- fixed huffman + + for (i=0; i < stbiw__ZHASH; ++i) + hash_table[i] = NULL; + + i=0; + while (i < data_len-3) { + // hash next 3 bytes of data to be compressed + int h = stbiw__zhash(data+i)&(stbiw__ZHASH-1), best=3; + unsigned char *bestloc = 0; + unsigned char **hlist = hash_table[h]; + int n = stbiw__sbcount(hlist); + for (j=0; j < n; ++j) { + if (hlist[j]-data > i-32768) { // if entry lies within window + int d = stbiw__zlib_countm(hlist[j], data+i, data_len-i); + if (d >= best) { best=d; bestloc=hlist[j]; } + } + } + // when hash table entry is too long, delete half the entries + if (hash_table[h] && stbiw__sbn(hash_table[h]) == 2*quality) { + STBIW_MEMMOVE(hash_table[h], hash_table[h]+quality, sizeof(hash_table[h][0])*quality); + stbiw__sbn(hash_table[h]) = quality; + } + stbiw__sbpush(hash_table[h],data+i); + + if (bestloc) { + // "lazy matching" - check match at *next* byte, and if it's better, do cur byte as literal + h = stbiw__zhash(data+i+1)&(stbiw__ZHASH-1); + hlist = hash_table[h]; + n = stbiw__sbcount(hlist); + for (j=0; j < n; ++j) { + if (hlist[j]-data > i-32767) { + int e = stbiw__zlib_countm(hlist[j], data+i+1, data_len-i-1); + if (e > best) { // if next match is better, bail on current match + bestloc = NULL; + break; + } + } + } + } + + if (bestloc) { + int d = (int) (data+i - bestloc); // distance back + STBIW_ASSERT(d <= 32767 && best <= 258); + for (j=0; best > lengthc[j+1]-1; ++j); + stbiw__zlib_huff(j+257); + if (lengtheb[j]) stbiw__zlib_add(best - lengthc[j], lengtheb[j]); + for (j=0; d > distc[j+1]-1; ++j); + stbiw__zlib_add(stbiw__zlib_bitrev(j,5),5); + if (disteb[j]) stbiw__zlib_add(d - distc[j], disteb[j]); + i += best; + } else { + stbiw__zlib_huffb(data[i]); + ++i; + } + } + // write out final bytes + for (;i < data_len; ++i) + stbiw__zlib_huffb(data[i]); + stbiw__zlib_huff(256); // end of block + // pad with 0 bits to byte boundary + while (bitcount) + stbiw__zlib_add(0,1); + + for (i=0; i < stbiw__ZHASH; ++i) + (void) stbiw__sbfree(hash_table[i]); + STBIW_FREE(hash_table); + + { + // compute adler32 on input + unsigned int s1=1, s2=0; + int blocklen = (int) (data_len % 5552); + j=0; + while (j < data_len) { + for (i=0; i < blocklen; ++i) { s1 += data[j+i]; s2 += s1; } + s1 %= 65521; s2 %= 65521; + j += blocklen; + blocklen = 5552; + } + stbiw__sbpush(out, STBIW_UCHAR(s2 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s2)); + stbiw__sbpush(out, STBIW_UCHAR(s1 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s1)); + } + *out_len = stbiw__sbn(out); + // make returned pointer freeable + STBIW_MEMMOVE(stbiw__sbraw(out), out, *out_len); + return (unsigned char *) stbiw__sbraw(out); +#endif // STBIW_ZLIB_COMPRESS +} + +static unsigned int stbiw__crc32(unsigned char *buffer, int len) +{ +#ifdef STBIW_CRC32 + return STBIW_CRC32(buffer, len); +#else + static unsigned int crc_table[256] = + { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0eDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D + }; + + unsigned int crc = ~0u; + int i; + for (i=0; i < len; ++i) + crc = (crc >> 8) ^ crc_table[buffer[i] ^ (crc & 0xff)]; + return ~crc; +#endif +} + +#define stbiw__wpng4(o,a,b,c,d) ((o)[0]=STBIW_UCHAR(a),(o)[1]=STBIW_UCHAR(b),(o)[2]=STBIW_UCHAR(c),(o)[3]=STBIW_UCHAR(d),(o)+=4) +#define stbiw__wp32(data,v) stbiw__wpng4(data, (v)>>24,(v)>>16,(v)>>8,(v)); +#define stbiw__wptag(data,s) stbiw__wpng4(data, s[0],s[1],s[2],s[3]) + +static void stbiw__wpcrc(unsigned char **data, int len) +{ + unsigned int crc = stbiw__crc32(*data - len - 4, len+4); + stbiw__wp32(*data, crc); +} + +static unsigned char stbiw__paeth(int a, int b, int c) +{ + int p = a + b - c, pa = abs(p-a), pb = abs(p-b), pc = abs(p-c); + if (pa <= pb && pa <= pc) return STBIW_UCHAR(a); + if (pb <= pc) return STBIW_UCHAR(b); + return STBIW_UCHAR(c); +} + +// @OPTIMIZE: provide an option that always forces left-predict or paeth predict +static void stbiw__encode_png_line(unsigned char *pixels, int stride_bytes, int width, int height, int y, int n, int filter_type, signed char *line_buffer) +{ + static int mapping[] = { 0,1,2,3,4 }; + static int firstmap[] = { 0,1,0,5,6 }; + int *mymap = (y != 0) ? mapping : firstmap; + int i; + int type = mymap[filter_type]; + unsigned char *z = pixels + stride_bytes * (stbi__flip_vertically_on_write ? height-1-y : y); + int signed_stride = stbi__flip_vertically_on_write ? -stride_bytes : stride_bytes; + + if (type==0) { + memcpy(line_buffer, z, width*n); + return; + } + + // first loop isn't optimized since it's just one pixel + for (i = 0; i < n; ++i) { + switch (type) { + case 1: line_buffer[i] = z[i]; break; + case 2: line_buffer[i] = z[i] - z[i-signed_stride]; break; + case 3: line_buffer[i] = z[i] - (z[i-signed_stride]>>1); break; + case 4: line_buffer[i] = (signed char) (z[i] - stbiw__paeth(0,z[i-signed_stride],0)); break; + case 5: line_buffer[i] = z[i]; break; + case 6: line_buffer[i] = z[i]; break; + } + } + switch (type) { + case 1: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-n]; break; + case 2: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-signed_stride]; break; + case 3: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - ((z[i-n] + z[i-signed_stride])>>1); break; + case 4: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], z[i-signed_stride], z[i-signed_stride-n]); break; + case 5: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - (z[i-n]>>1); break; + case 6: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], 0,0); break; + } +} + +STBIWDEF unsigned char *stbi_write_png_to_mem(const unsigned char *pixels, int stride_bytes, int x, int y, int n, int *out_len) +{ + int force_filter = stbi_write_force_png_filter; + int ctype[5] = { -1, 0, 4, 2, 6 }; + unsigned char sig[8] = { 137,80,78,71,13,10,26,10 }; + unsigned char *out,*o, *filt, *zlib; + signed char *line_buffer; + int j,zlen; + + if (stride_bytes == 0) + stride_bytes = x * n; + + if (force_filter >= 5) { + force_filter = -1; + } + + filt = (unsigned char *) STBIW_MALLOC((x*n+1) * y); if (!filt) return 0; + line_buffer = (signed char *) STBIW_MALLOC(x * n); if (!line_buffer) { STBIW_FREE(filt); return 0; } + for (j=0; j < y; ++j) { + int filter_type; + if (force_filter > -1) { + filter_type = force_filter; + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, force_filter, line_buffer); + } else { // Estimate the best filter by running through all of them: + int best_filter = 0, best_filter_val = 0x7fffffff, est, i; + for (filter_type = 0; filter_type < 5; filter_type++) { + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, filter_type, line_buffer); + + // Estimate the entropy of the line using this filter; the less, the better. + est = 0; + for (i = 0; i < x*n; ++i) { + est += abs((signed char) line_buffer[i]); + } + if (est < best_filter_val) { + best_filter_val = est; + best_filter = filter_type; + } + } + if (filter_type != best_filter) { // If the last iteration already got us the best filter, don't redo it + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, best_filter, line_buffer); + filter_type = best_filter; + } + } + // when we get here, filter_type contains the filter type, and line_buffer contains the data + filt[j*(x*n+1)] = (unsigned char) filter_type; + STBIW_MEMMOVE(filt+j*(x*n+1)+1, line_buffer, x*n); + } + STBIW_FREE(line_buffer); + zlib = stbi_zlib_compress(filt, y*( x*n+1), &zlen, stbi_write_png_compression_level); + STBIW_FREE(filt); + if (!zlib) return 0; + + // each tag requires 12 bytes of overhead + out = (unsigned char *) STBIW_MALLOC(8 + 12+13 + 12+zlen + 12); + if (!out) return 0; + *out_len = 8 + 12+13 + 12+zlen + 12; + + o=out; + STBIW_MEMMOVE(o,sig,8); o+= 8; + stbiw__wp32(o, 13); // header length + stbiw__wptag(o, "IHDR"); + stbiw__wp32(o, x); + stbiw__wp32(o, y); + *o++ = 8; + *o++ = STBIW_UCHAR(ctype[n]); + *o++ = 0; + *o++ = 0; + *o++ = 0; + stbiw__wpcrc(&o,13); + + stbiw__wp32(o, zlen); + stbiw__wptag(o, "IDAT"); + STBIW_MEMMOVE(o, zlib, zlen); + o += zlen; + STBIW_FREE(zlib); + stbiw__wpcrc(&o, zlen); + + stbiw__wp32(o,0); + stbiw__wptag(o, "IEND"); + stbiw__wpcrc(&o,0); + + STBIW_ASSERT(o == out + *out_len); + + return out; +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int x, int y, int comp, const void *data, int stride_bytes) +{ + FILE *f; + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); + if (png == NULL) return 0; + + f = stbiw__fopen(filename, "wb"); + if (!f) { STBIW_FREE(png); return 0; } + fwrite(png, 1, len, f); + fclose(f); + STBIW_FREE(png); + return 1; +} +#endif + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int stride_bytes) +{ + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); + if (png == NULL) return 0; + func(context, png, len); + STBIW_FREE(png); + return 1; +} + + +/* *************************************************************************** + * + * JPEG writer + * + * This is based on Jon Olick's jo_jpeg.cpp: + * public domain Simple, Minimalistic JPEG writer - http://www.jonolick.com/code.html + */ + +static const unsigned char stbiw__jpg_ZigZag[] = { 0,1,5,6,14,15,27,28,2,4,7,13,16,26,29,42,3,8,12,17,25,30,41,43,9,11,18, + 24,31,40,44,53,10,19,23,32,39,45,52,54,20,22,33,38,46,51,55,60,21,34,37,47,50,56,59,61,35,36,48,49,57,58,62,63 }; + +static void stbiw__jpg_writeBits(stbi__write_context *s, int *bitBufP, int *bitCntP, const unsigned short *bs) { + int bitBuf = *bitBufP, bitCnt = *bitCntP; + bitCnt += bs[1]; + bitBuf |= bs[0] << (24 - bitCnt); + while(bitCnt >= 8) { + unsigned char c = (bitBuf >> 16) & 255; + stbiw__putc(s, c); + if(c == 255) { + stbiw__putc(s, 0); + } + bitBuf <<= 8; + bitCnt -= 8; + } + *bitBufP = bitBuf; + *bitCntP = bitCnt; +} + +static void stbiw__jpg_DCT(float *d0p, float *d1p, float *d2p, float *d3p, float *d4p, float *d5p, float *d6p, float *d7p) { + float d0 = *d0p, d1 = *d1p, d2 = *d2p, d3 = *d3p, d4 = *d4p, d5 = *d5p, d6 = *d6p, d7 = *d7p; + float z1, z2, z3, z4, z5, z11, z13; + + float tmp0 = d0 + d7; + float tmp7 = d0 - d7; + float tmp1 = d1 + d6; + float tmp6 = d1 - d6; + float tmp2 = d2 + d5; + float tmp5 = d2 - d5; + float tmp3 = d3 + d4; + float tmp4 = d3 - d4; + + // Even part + float tmp10 = tmp0 + tmp3; // phase 2 + float tmp13 = tmp0 - tmp3; + float tmp11 = tmp1 + tmp2; + float tmp12 = tmp1 - tmp2; + + d0 = tmp10 + tmp11; // phase 3 + d4 = tmp10 - tmp11; + + z1 = (tmp12 + tmp13) * 0.707106781f; // c4 + d2 = tmp13 + z1; // phase 5 + d6 = tmp13 - z1; + + // Odd part + tmp10 = tmp4 + tmp5; // phase 2 + tmp11 = tmp5 + tmp6; + tmp12 = tmp6 + tmp7; + + // The rotator is modified from fig 4-8 to avoid extra negations. + z5 = (tmp10 - tmp12) * 0.382683433f; // c6 + z2 = tmp10 * 0.541196100f + z5; // c2-c6 + z4 = tmp12 * 1.306562965f + z5; // c2+c6 + z3 = tmp11 * 0.707106781f; // c4 + + z11 = tmp7 + z3; // phase 5 + z13 = tmp7 - z3; + + *d5p = z13 + z2; // phase 6 + *d3p = z13 - z2; + *d1p = z11 + z4; + *d7p = z11 - z4; + + *d0p = d0; *d2p = d2; *d4p = d4; *d6p = d6; +} + +static void stbiw__jpg_calcBits(int val, unsigned short bits[2]) { + int tmp1 = val < 0 ? -val : val; + val = val < 0 ? val-1 : val; + bits[1] = 1; + while(tmp1 >>= 1) { + ++bits[1]; + } + bits[0] = val & ((1<0)&&(DU[end0pos]==0); --end0pos) { + } + // end0pos = first element in reverse order !=0 + if(end0pos == 0) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + return DU[0]; + } + for(i = 1; i <= end0pos; ++i) { + int startpos = i; + int nrzeroes; + unsigned short bits[2]; + for (; DU[i]==0 && i<=end0pos; ++i) { + } + nrzeroes = i-startpos; + if ( nrzeroes >= 16 ) { + int lng = nrzeroes>>4; + int nrmarker; + for (nrmarker=1; nrmarker <= lng; ++nrmarker) + stbiw__jpg_writeBits(s, bitBuf, bitCnt, M16zeroes); + nrzeroes &= 15; + } + stbiw__jpg_calcBits(DU[i], bits); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, HTAC[(nrzeroes<<4)+bits[1]]); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, bits); + } + if(end0pos != 63) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + } + return DU[0]; +} + +static int stbi_write_jpg_core(stbi__write_context *s, int width, int height, int comp, const void* data, int quality) { + // Constants that don't pollute global namespace + static const unsigned char std_dc_luminance_nrcodes[] = {0,0,1,5,1,1,1,1,1,1,0,0,0,0,0,0,0}; + static const unsigned char std_dc_luminance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; + static const unsigned char std_ac_luminance_nrcodes[] = {0,0,2,1,3,3,2,4,3,5,5,4,4,0,0,1,0x7d}; + static const unsigned char std_ac_luminance_values[] = { + 0x01,0x02,0x03,0x00,0x04,0x11,0x05,0x12,0x21,0x31,0x41,0x06,0x13,0x51,0x61,0x07,0x22,0x71,0x14,0x32,0x81,0x91,0xa1,0x08, + 0x23,0x42,0xb1,0xc1,0x15,0x52,0xd1,0xf0,0x24,0x33,0x62,0x72,0x82,0x09,0x0a,0x16,0x17,0x18,0x19,0x1a,0x25,0x26,0x27,0x28, + 0x29,0x2a,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58,0x59, + 0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x83,0x84,0x85,0x86,0x87,0x88,0x89, + 0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4,0xb5,0xb6, + 0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xe1,0xe2, + 0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa + }; + static const unsigned char std_dc_chrominance_nrcodes[] = {0,0,3,1,1,1,1,1,1,1,1,1,0,0,0,0,0}; + static const unsigned char std_dc_chrominance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; + static const unsigned char std_ac_chrominance_nrcodes[] = {0,0,2,1,2,4,4,3,4,7,5,4,4,0,1,2,0x77}; + static const unsigned char std_ac_chrominance_values[] = { + 0x00,0x01,0x02,0x03,0x11,0x04,0x05,0x21,0x31,0x06,0x12,0x41,0x51,0x07,0x61,0x71,0x13,0x22,0x32,0x81,0x08,0x14,0x42,0x91, + 0xa1,0xb1,0xc1,0x09,0x23,0x33,0x52,0xf0,0x15,0x62,0x72,0xd1,0x0a,0x16,0x24,0x34,0xe1,0x25,0xf1,0x17,0x18,0x19,0x1a,0x26, + 0x27,0x28,0x29,0x2a,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58, + 0x59,0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x82,0x83,0x84,0x85,0x86,0x87, + 0x88,0x89,0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4, + 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda, + 0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa + }; + // Huffman tables + static const unsigned short YDC_HT[256][2] = { {0,2},{2,3},{3,3},{4,3},{5,3},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9}}; + static const unsigned short UVDC_HT[256][2] = { {0,2},{1,2},{2,2},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9},{1022,10},{2046,11}}; + static const unsigned short YAC_HT[256][2] = { + {10,4},{0,2},{1,2},{4,3},{11,4},{26,5},{120,7},{248,8},{1014,10},{65410,16},{65411,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {12,4},{27,5},{121,7},{502,9},{2038,11},{65412,16},{65413,16},{65414,16},{65415,16},{65416,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {28,5},{249,8},{1015,10},{4084,12},{65417,16},{65418,16},{65419,16},{65420,16},{65421,16},{65422,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {58,6},{503,9},{4085,12},{65423,16},{65424,16},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {59,6},{1016,10},{65430,16},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {122,7},{2039,11},{65438,16},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {123,7},{4086,12},{65446,16},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {250,8},{4087,12},{65454,16},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {504,9},{32704,15},{65462,16},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {505,9},{65470,16},{65471,16},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {506,9},{65479,16},{65480,16},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1017,10},{65488,16},{65489,16},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1018,10},{65497,16},{65498,16},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2040,11},{65506,16},{65507,16},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {65515,16},{65516,16},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2041,11},{65525,16},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} + }; + static const unsigned short UVAC_HT[256][2] = { + {0,2},{1,2},{4,3},{10,4},{24,5},{25,5},{56,6},{120,7},{500,9},{1014,10},{4084,12},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {11,4},{57,6},{246,8},{501,9},{2038,11},{4085,12},{65416,16},{65417,16},{65418,16},{65419,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {26,5},{247,8},{1015,10},{4086,12},{32706,15},{65420,16},{65421,16},{65422,16},{65423,16},{65424,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {27,5},{248,8},{1016,10},{4087,12},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{65430,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {58,6},{502,9},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{65438,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {59,6},{1017,10},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{65446,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {121,7},{2039,11},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{65454,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {122,7},{2040,11},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{65462,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {249,8},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{65470,16},{65471,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {503,9},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{65479,16},{65480,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {504,9},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{65488,16},{65489,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {505,9},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{65497,16},{65498,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {506,9},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{65506,16},{65507,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2041,11},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{65515,16},{65516,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {16352,14},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{65525,16},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1018,10},{32707,15},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} + }; + static const int YQT[] = {16,11,10,16,24,40,51,61,12,12,14,19,26,58,60,55,14,13,16,24,40,57,69,56,14,17,22,29,51,87,80,62,18,22, + 37,56,68,109,103,77,24,35,55,64,81,104,113,92,49,64,78,87,103,121,120,101,72,92,95,98,112,100,103,99}; + static const int UVQT[] = {17,18,24,47,99,99,99,99,18,21,26,66,99,99,99,99,24,26,56,99,99,99,99,99,47,66,99,99,99,99,99,99, + 99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99}; + static const float aasf[] = { 1.0f * 2.828427125f, 1.387039845f * 2.828427125f, 1.306562965f * 2.828427125f, 1.175875602f * 2.828427125f, + 1.0f * 2.828427125f, 0.785694958f * 2.828427125f, 0.541196100f * 2.828427125f, 0.275899379f * 2.828427125f }; + + int row, col, i, k, subsample; + float fdtbl_Y[64], fdtbl_UV[64]; + unsigned char YTable[64], UVTable[64]; + + if(!data || !width || !height || comp > 4 || comp < 1) { + return 0; + } + + quality = quality ? quality : 90; + subsample = quality <= 90 ? 1 : 0; + quality = quality < 1 ? 1 : quality > 100 ? 100 : quality; + quality = quality < 50 ? 5000 / quality : 200 - quality * 2; + + for(i = 0; i < 64; ++i) { + int uvti, yti = (YQT[i]*quality+50)/100; + YTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (yti < 1 ? 1 : yti > 255 ? 255 : yti); + uvti = (UVQT[i]*quality+50)/100; + UVTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (uvti < 1 ? 1 : uvti > 255 ? 255 : uvti); + } + + for(row = 0, k = 0; row < 8; ++row) { + for(col = 0; col < 8; ++col, ++k) { + fdtbl_Y[k] = 1 / (YTable [stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + fdtbl_UV[k] = 1 / (UVTable[stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + } + } + + // Write Headers + { + static const unsigned char head0[] = { 0xFF,0xD8,0xFF,0xE0,0,0x10,'J','F','I','F',0,1,1,0,0,1,0,1,0,0,0xFF,0xDB,0,0x84,0 }; + static const unsigned char head2[] = { 0xFF,0xDA,0,0xC,3,1,0,2,0x11,3,0x11,0,0x3F,0 }; + const unsigned char head1[] = { 0xFF,0xC0,0,0x11,8,(unsigned char)(height>>8),STBIW_UCHAR(height),(unsigned char)(width>>8),STBIW_UCHAR(width), + 3,1,(unsigned char)(subsample?0x22:0x11),0,2,0x11,1,3,0x11,1,0xFF,0xC4,0x01,0xA2,0 }; + s->func(s->context, (void*)head0, sizeof(head0)); + s->func(s->context, (void*)YTable, sizeof(YTable)); + stbiw__putc(s, 1); + s->func(s->context, UVTable, sizeof(UVTable)); + s->func(s->context, (void*)head1, sizeof(head1)); + s->func(s->context, (void*)(std_dc_luminance_nrcodes+1), sizeof(std_dc_luminance_nrcodes)-1); + s->func(s->context, (void*)std_dc_luminance_values, sizeof(std_dc_luminance_values)); + stbiw__putc(s, 0x10); // HTYACinfo + s->func(s->context, (void*)(std_ac_luminance_nrcodes+1), sizeof(std_ac_luminance_nrcodes)-1); + s->func(s->context, (void*)std_ac_luminance_values, sizeof(std_ac_luminance_values)); + stbiw__putc(s, 1); // HTUDCinfo + s->func(s->context, (void*)(std_dc_chrominance_nrcodes+1), sizeof(std_dc_chrominance_nrcodes)-1); + s->func(s->context, (void*)std_dc_chrominance_values, sizeof(std_dc_chrominance_values)); + stbiw__putc(s, 0x11); // HTUACinfo + s->func(s->context, (void*)(std_ac_chrominance_nrcodes+1), sizeof(std_ac_chrominance_nrcodes)-1); + s->func(s->context, (void*)std_ac_chrominance_values, sizeof(std_ac_chrominance_values)); + s->func(s->context, (void*)head2, sizeof(head2)); + } + + // Encode 8x8 macroblocks + { + static const unsigned short fillBits[] = {0x7F, 7}; + int DCY=0, DCU=0, DCV=0; + int bitBuf=0, bitCnt=0; + // comp == 2 is grey+alpha (alpha is ignored) + int ofsG = comp > 2 ? 1 : 0, ofsB = comp > 2 ? 2 : 0; + const unsigned char *dataR = (const unsigned char *)data; + const unsigned char *dataG = dataR + ofsG; + const unsigned char *dataB = dataR + ofsB; + int x, y, pos; + if(subsample) { + for(y = 0; y < height; y += 16) { + for(x = 0; x < width; x += 16) { + float Y[256], U[256], V[256]; + for(row = y, pos = 0; row < y+16; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp; + for(col = x; col < x+16; ++col, ++pos) { + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width-1))*comp; + float r = dataR[p], g = dataG[p], b = dataB[p]; + Y[pos]= +0.29900f*r + 0.58700f*g + 0.11400f*b - 128; + U[pos]= -0.16874f*r - 0.33126f*g + 0.50000f*b; + V[pos]= +0.50000f*r - 0.41869f*g - 0.08131f*b; + } + } + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+0, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+8, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+128, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+136, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + + // subsample U,V + { + float subU[64], subV[64]; + int yy, xx; + for(yy = 0, pos = 0; yy < 8; ++yy) { + for(xx = 0; xx < 8; ++xx, ++pos) { + int j = yy*32+xx*2; + subU[pos] = (U[j+0] + U[j+1] + U[j+16] + U[j+17]) * 0.25f; + subV[pos] = (V[j+0] + V[j+1] + V[j+16] + V[j+17]) * 0.25f; + } + } + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subU, 8, fdtbl_UV, DCU, UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subV, 8, fdtbl_UV, DCV, UVDC_HT, UVAC_HT); + } + } + } + } else { + for(y = 0; y < height; y += 8) { + for(x = 0; x < width; x += 8) { + float Y[64], U[64], V[64]; + for(row = y, pos = 0; row < y+8; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp; + for(col = x; col < x+8; ++col, ++pos) { + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width-1))*comp; + float r = dataR[p], g = dataG[p], b = dataB[p]; + Y[pos]= +0.29900f*r + 0.58700f*g + 0.11400f*b - 128; + U[pos]= -0.16874f*r - 0.33126f*g + 0.50000f*b; + V[pos]= +0.50000f*r - 0.41869f*g - 0.08131f*b; + } + } + + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y, 8, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, U, 8, fdtbl_UV, DCU, UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, V, 8, fdtbl_UV, DCV, UVDC_HT, UVAC_HT); + } + } + } + + // Do the bit alignment of the EOI marker + stbiw__jpg_writeBits(s, &bitBuf, &bitCnt, fillBits); + } + + // EOI + stbiw__putc(s, 0xFF); + stbiw__putc(s, 0xD9); + + return 1; +} + +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality) +{ + stbi__write_context s; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_jpg_core(&s, x, y, comp, (void *) data, quality); +} + + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality) +{ + stbi__write_context s; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_jpg_core(&s, x, y, comp, data, quality); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +#endif // STB_IMAGE_WRITE_IMPLEMENTATION + +/* Revision history + 1.14 (2020-02-02) updated JPEG writer to downsample chroma channels + 1.13 + 1.12 + 1.11 (2019-08-11) + + 1.10 (2019-02-07) + support utf8 filenames in Windows; fix warnings and platform ifdefs + 1.09 (2018-02-11) + fix typo in zlib quality API, improve STB_I_W_STATIC in C++ + 1.08 (2018-01-29) + add stbi__flip_vertically_on_write, external zlib, zlib quality, choose PNG filter + 1.07 (2017-07-24) + doc fix + 1.06 (2017-07-23) + writing JPEG (using Jon Olick's code) + 1.05 ??? + 1.04 (2017-03-03) + monochrome BMP expansion + 1.03 ??? + 1.02 (2016-04-02) + avoid allocating large structures on the stack + 1.01 (2016-01-16) + STBIW_REALLOC_SIZED: support allocators with no realloc support + avoid race-condition in crc initialization + minor compile issues + 1.00 (2015-09-14) + installable file IO function + 0.99 (2015-09-13) + warning fixes; TGA rle support + 0.98 (2015-04-08) + added STBIW_MALLOC, STBIW_ASSERT etc + 0.97 (2015-01-18) + fixed HDR asserts, rewrote HDR rle logic + 0.96 (2015-01-17) + add HDR output + fix monochrome BMP + 0.95 (2014-08-17) + add monochrome TGA output + 0.94 (2014-05-31) + rename private functions to avoid conflicts with stb_image.h + 0.93 (2014-05-27) + warning fixes + 0.92 (2010-08-01) + casts to unsigned char to fix warnings + 0.91 (2010-07-17) + first public release + 0.90 first internal release +*/ + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/src/external/stb/stb_image_write.h.license b/src/external/stb/stb_image_write.h.license new file mode 100644 index 000000000..a27e204e9 --- /dev/null +++ b/src/external/stb/stb_image_write.h.license @@ -0,0 +1,3 @@ +Copyright (c) 2017 Sean Barrett + +SPDX-License-Identifier: MIT OR Unlicense diff --git a/src/xrt/.editorconfig b/src/xrt/.editorconfig new file mode 100644 index 000000000..8746b3cb6 --- /dev/null +++ b/src/xrt/.editorconfig @@ -0,0 +1,10 @@ +# To use this config on your editor, follow the instructions at: +# http://editorconfig.org + +# SPDX-License-Identifier: CC0-1.0 +# SPDX-FileCopyrightText: 2018-2021 Collabora, Ltd. and the Monado contributors + +[*] +charset = utf-8 +insert_final_newline = true +trim_trailing_whitespace = true diff --git a/src/xrt/auxiliary/CMakeLists.txt b/src/xrt/auxiliary/CMakeLists.txt index 56f1f7875..749c8948a 100644 --- a/src/xrt/auxiliary/CMakeLists.txt +++ b/src/xrt/auxiliary/CMakeLists.txt @@ -1,12 +1,14 @@ # Copyright 2019-2020, Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 +add_subdirectory(bindings) + set(ANDROID_SOURCE_FILES android/android_ahardwarebuffer_allocator.c android/android_ahardwarebuffer_allocator.h android/android_custom_surface.cpp android/android_custom_surface.h - android/android_globals.c + android/android_globals.cpp android/android_globals.h android/android_load_class.cpp android/android_load_class.hpp @@ -21,6 +23,8 @@ set(MATH_SOURCE_FILES math/m_eigen_interop.hpp math/m_filter_fifo.c math/m_filter_fifo.h + math/m_filter_one_euro.c + math/m_filter_one_euro.h math/m_hash.cpp math/m_imu_3dof.c math/m_imu_3dof.h @@ -32,6 +36,8 @@ set(MATH_SOURCE_FILES math/m_predict.c math/m_predict.h math/m_quatexpmap.cpp + math/m_relation_history.h + math/m_relation_history.cpp math/m_space.cpp math/m_space.h math/m_vec2.h @@ -69,6 +75,14 @@ if(XRT_HAVE_DBUS) ) endif() +set(GSTREAMER_SOURCE_FILES + gstreamer/gst_internal.h + gstreamer/gst_sink.h + gstreamer/gst_sink.c + gstreamer/gst_pipeline.h + gstreamer/gst_pipeline.c + ) + set(TRACKING_SOURCE_FILES tracking/t_data_utils.c tracking/t_imu_fusion.hpp @@ -87,6 +101,8 @@ if(XRT_HAVE_OPENCV) tracking/t_debug_hsv_picker.cpp tracking/t_debug_hsv_viewer.cpp tracking/t_file.cpp + tracking/t_frame_cv_mat_wrapper.cpp + tracking/t_frame_cv_mat_wrapper.hpp tracking/t_fusion.hpp tracking/t_helper_debug_sink.hpp tracking/t_hsv_filter.c @@ -96,6 +112,10 @@ if(XRT_HAVE_OPENCV) tracking/t_tracker_psvr.cpp tracking/t_tracker_hand.cpp ) + + if(XRT_HAVE_SLAM) + list(APPEND TRACKING_SOURCE_FILES tracking/t_tracker_slam.cpp) + endif() endif() set(UTIL_SOURCE_FILES @@ -111,12 +131,16 @@ set(UTIL_SOURCE_FILES util/u_distortion_mesh.h util/u_documentation.h util/u_file.c + util/u_file.cpp util/u_file.h util/u_format.c util/u_format.h util/u_frame.c util/u_frame.h + util/u_generic_callbacks.hpp util/u_git_tag.h + util/u_hand_tracking.c + util/u_hand_tracking.h util/u_handles.c util/u_handles.h util/u_hashmap.cpp @@ -129,20 +153,28 @@ set(UTIL_SOURCE_FILES util/u_logging.h util/u_misc.c util/u_misc.h - util/u_render_timing.c - util/u_render_timing.h util/u_sink.h util/u_sink_converter.c util/u_sink_deinterleaver.c util/u_sink_queue.c util/u_sink_quirk.c util/u_sink_split.c + util/u_template_historybuf.hpp util/u_time.cpp util/u_time.h + util/u_timing.h + util/u_timing_fake.c + util/u_timing_frame.c + util/u_timing_render.c + util/u_trace_marker.c + util/u_trace_marker.h util/u_var.cpp util/u_var.h - util/u_hand_tracking.c - util/u_hand_tracking.h + util/u_config_json.c + util/u_config_json.h + util/u_verify.h + util/u_process.c + util/u_process.h ) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/util/u_git_tag.c.in" "${CMAKE_CURRENT_BINARY_DIR}/u_git_tag.c" @ONLY) @@ -186,7 +218,7 @@ endif() # Math library. add_library(aux_math STATIC ${MATH_SOURCE_FILES}) -target_link_libraries(aux_math PUBLIC aux-includes) +target_link_libraries(aux_math PUBLIC aux-includes aux_util) # Math files has extra include(s). target_include_directories(aux_math SYSTEM @@ -195,7 +227,7 @@ target_include_directories(aux_math SYSTEM # Util library. add_library(aux_util STATIC ${UTIL_SOURCE_FILES}) -target_link_libraries(aux_util PUBLIC aux-includes xrt-pthreads) +target_link_libraries(aux_util PUBLIC aux-includes xrt-pthreads aux_generated_bindings) # for u_device target_link_libraries(aux_util PUBLIC aux_math) if(XRT_HAVE_JPEG) @@ -209,13 +241,39 @@ if(XRT_HAVE_SYSTEM_CJSON) else() target_link_libraries(aux_util PUBLIC xrt-external-cjson) endif() +# For u_trace_marker +if(XRT_FEATURE_TRACING AND XRT_HAVE_PERCETTO) + target_link_libraries(aux_util PUBLIC Percetto::percetto) +endif() +if(XRT_HAVE_LIBBSD) + target_include_directories(aux_util SYSTEM PRIVATE ${LIBBSD_INCLUDE_DIRS}) + target_link_libraries(aux_util PUBLIC ${LIBBSD_LIBRARIES}) +endif() if(ANDROID) target_link_libraries(aux_util PUBLIC ${ANDROID_LOG_LIBRARY}) endif() + +# GStreamer library. +if(XRT_HAVE_GST) + add_library(aux_gstreamer STATIC ${GSTREAMER_SOURCE_FILES}) + target_link_libraries(aux_gstreamer PUBLIC + aux-includes + ) + target_link_libraries(aux_gstreamer PRIVATE + xrt-interfaces + aux_math + aux_os + ${GST_LIBRARIES} + ) + target_include_directories(aux_gstreamer PRIVATE + ${GST_INCLUDE_DIRS} + ) +endif() + # Tracking library. add_library(aux_tracking STATIC ${TRACKING_SOURCE_FILES}) -target_link_libraries(aux_tracking PUBLIC aux-includes PRIVATE aux_math) +target_link_libraries(aux_tracking PUBLIC aux-includes PRIVATE aux_math aux_util) # Tracking files have extra includes. target_include_directories(aux_tracking SYSTEM @@ -232,12 +290,26 @@ if(XRT_HAVE_OPENCV) ${OpenCV_INCLUDE_DIRS} ) target_link_libraries(aux_tracking PUBLIC ${OpenCV_LIBRARIES}) + if(XRT_HAVE_SLAM) + target_link_libraries(aux_tracking PRIVATE xrt-external-slam) + endif() +endif() + +if (XRT_BUILD_DRIVER_VIVE OR XRT_BUILD_DRIVER_SURVIVE) + set(VIVE_CONFIG_SOURCE_FILES + vive/vive_config.h + vive/vive_config.c + ) + add_library(aux_vive STATIC ${VIVE_CONFIG_SOURCE_FILES}) + target_link_libraries(aux_vive PRIVATE xrt-interfaces aux_util aux_math aux_tracking xrt-external-cjson) + target_link_libraries(aux_vive PRIVATE ${ZLIB_LIBRARIES}) + target_include_directories(aux_vive PRIVATE ${ZLIB_INCLUDE_DIRS}) endif() if(XRT_HAVE_VULKAN) # Vulkan library. add_library(aux_vk STATIC ${VK_SOURCE_FILES}) - target_link_libraries(aux_vk PUBLIC aux-includes) + target_link_libraries(aux_vk PUBLIC aux_os aux_util) target_link_libraries(aux_vk PUBLIC Vulkan::Vulkan) target_include_directories(aux_vk PUBLIC ${Vulkan_INCLUDE_DIR}) if(ANDROID) @@ -252,11 +324,5 @@ if(ANDROID) PUBLIC aux_util PRIVATE ${ANDROID_LIBRARY} ${ANDROID_LOG_LIBRARY} xrt-external-jni-wrap xrt-external-jnipp ) - set_target_properties(aux_android - PROPERTIES - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON) target_link_libraries(aux_vk PUBLIC aux_android) endif() - -add_subdirectory(bindings) diff --git a/src/xrt/auxiliary/android/android_custom_surface.cpp b/src/xrt/auxiliary/android/android_custom_surface.cpp index 829f01fd4..eacfabe1c 100644 --- a/src/xrt/auxiliary/android/android_custom_surface.cpp +++ b/src/xrt/auxiliary/android/android_custom_surface.cpp @@ -24,6 +24,8 @@ using wrap::android::app::Activity; using wrap::android::view::SurfaceHolder; using wrap::org::freedesktop::monado::auxiliary::MonadoView; +using xrt::auxiliary::android::getAppInfo; +using xrt::auxiliary::android::loadClassFromPackage; struct android_custom_surface diff --git a/src/xrt/auxiliary/android/android_globals.c b/src/xrt/auxiliary/android/android_globals.cpp similarity index 68% rename from src/xrt/auxiliary/android/android_globals.c rename to src/xrt/auxiliary/android/android_globals.cpp index 9bf33bdf9..fd7b68f8b 100644 --- a/src/xrt/auxiliary/android/android_globals.c +++ b/src/xrt/auxiliary/android/android_globals.cpp @@ -10,6 +10,7 @@ #include "android_globals.h" #include +#include /*! * @todo Do we need locking here? Do we need to create global refs for the @@ -17,11 +18,11 @@ */ static struct { - struct _JavaVM *vm; - void *activity; - void *context; - struct _ANativeWindow *window; -} android_globals = {NULL, NULL, NULL}; + struct _JavaVM *vm = nullptr; + void *activity = nullptr; + void *context = nullptr; + struct _ANativeWindow *window = nullptr; +} android_globals; void android_globals_store_vm_and_activity(struct _JavaVM *vm, void *activity) @@ -33,11 +34,21 @@ android_globals_store_vm_and_activity(struct _JavaVM *vm, void *activity) void android_globals_store_vm_and_context(struct _JavaVM *vm, void *context) { - android_globals.vm = vm; android_globals.context = context; + if (android_globals_is_instance_of_activity(vm, context)) { + android_globals.activity = context; + } } +bool +android_globals_is_instance_of_activity(struct _JavaVM *vm, void *obj) +{ + jni::init(vm); + + auto activity_cls = jni::Class(wrap::android::app::Activity::getTypeName()); + return JNI_TRUE == jni::env()->IsInstanceOf((jobject)obj, activity_cls.getHandle()); +} void android_globals_store_window(struct _ANativeWindow *window) { diff --git a/src/xrt/auxiliary/android/android_globals.h b/src/xrt/auxiliary/android/android_globals.h index 5257c7d1e..95f4abc72 100644 --- a/src/xrt/auxiliary/android/android_globals.h +++ b/src/xrt/auxiliary/android/android_globals.h @@ -34,6 +34,13 @@ android_globals_store_vm_and_activity(struct _JavaVM *vm, void *activity); void android_globals_store_vm_and_context(struct _JavaVM *vm, void *context); + +/*! + * Is the provided jobject an instance of android.app.Activity? + */ +bool +android_globals_is_instance_of_activity(struct _JavaVM *vm, void *obj); + /*! * Retrieve the Java VM pointer previously stored, if any. */ diff --git a/src/xrt/auxiliary/android/android_load_class.cpp b/src/xrt/auxiliary/android/android_load_class.cpp index 24b2a85f9..5db978ba4 100644 --- a/src/xrt/auxiliary/android/android_load_class.cpp +++ b/src/xrt/auxiliary/android/android_load_class.cpp @@ -12,7 +12,6 @@ #include "util/u_logging.h" #include "wrap/android.content.h" -#include "wrap/dalvik.system.h" #include "jni.h" @@ -20,6 +19,8 @@ using wrap::android::content::Context; using wrap::android::content::pm::ApplicationInfo; using wrap::android::content::pm::PackageManager; +namespace xrt::auxiliary::android { + ApplicationInfo getAppInfo(std::string const &packageName, jobject application_context) { @@ -79,6 +80,8 @@ loadClassFromPackage(ApplicationInfo applicationInfo, jobject application_contex return wrap::java::lang::Class(); } } +} // namespace xrt::auxiliary::android + void * android_load_class_from_package(struct _JavaVM *vm, @@ -86,6 +89,7 @@ android_load_class_from_package(struct _JavaVM *vm, void *application_context, const char *classname) { + using namespace xrt::auxiliary::android; jni::init(vm); Context context((jobject)application_context); auto info = getAppInfo(pkgname, (jobject)application_context); diff --git a/src/xrt/auxiliary/android/android_load_class.hpp b/src/xrt/auxiliary/android/android_load_class.hpp index 327158c3e..967889150 100644 --- a/src/xrt/auxiliary/android/android_load_class.hpp +++ b/src/xrt/auxiliary/android/android_load_class.hpp @@ -15,6 +15,9 @@ #ifdef XRT_OS_ANDROID +//! C++-only functionality in the Android auxiliary library +namespace xrt::auxiliary::android { + using wrap::android::content::pm::ApplicationInfo; ApplicationInfo @@ -23,4 +26,6 @@ getAppInfo(std::string const &packageName, jobject application_context); wrap::java::lang::Class loadClassFromPackage(ApplicationInfo applicationInfo, jobject application_context, const char *clazz_name); +} // namespace xrt::auxiliary::android + #endif // XRT_OS_ANDROID diff --git a/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.cpp b/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.cpp index 63d1f34ce..2d44d1149 100644 --- a/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.cpp +++ b/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.cpp @@ -1,4 +1,4 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -14,13 +14,15 @@ namespace wrap { namespace org::freedesktop::monado::auxiliary { MonadoView::Meta::Meta(jni::jclass clazz) : MetaBase(MonadoView::getTypeName(), clazz), - attachToActivity(classRef().getStaticMethod("attachToActivity", - "(Landroid/app/Activity;J)Lorg/freedesktop/" - "monado/auxiliary/MonadoView;")), - waitGetSurfaceHolder(classRef().getMethod("waitGetSurfaceHolder", "(I)Landroid/view/SurfaceHolder;")), - markAsDiscardedByNative(classRef().getMethod("markAsDiscardedByNative", "()V")), + attachToActivity(classRef().getStaticMethod( + "attachToActivity", "(Landroid/app/Activity;J)Lorg/freedesktop/monado/auxiliary/MonadoView;")), + attachToActivity1(classRef().getStaticMethod( + "attachToActivity", "(Landroid/app/Activity;)Lorg/freedesktop/monado/auxiliary/MonadoView;")), getDisplayMetrics(classRef().getStaticMethod("getDisplayMetrics", - "(Landroid/app/Activity;)Landroid/util/DisplayMetrics;")) + "(Landroid/app/Activity;)Landroid/util/DisplayMetrics;")), + getNativePointer(classRef().getMethod("getNativePointer", "()J")), + markAsDiscardedByNative(classRef().getMethod("markAsDiscardedByNative", "()V")), + waitGetSurfaceHolder(classRef().getMethod("waitGetSurfaceHolder", "(I)Landroid/view/SurfaceHolder;")) {} } // namespace org::freedesktop::monado::auxiliary } // namespace wrap diff --git a/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.hpp b/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.hpp index 1cea26e62..909c11988 100644 --- a/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.hpp +++ b/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.hpp @@ -1,4 +1,4 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -27,6 +27,7 @@ namespace org::freedesktop::monado::auxiliary { } // namespace wrap + namespace wrap { namespace org::freedesktop::monado::auxiliary { /*! @@ -42,32 +43,60 @@ namespace org::freedesktop::monado::auxiliary { return "org/freedesktop/monado/auxiliary/MonadoView"; } + static constexpr const char * + getFullyQualifiedTypeName() noexcept + { + return "org.freedesktop.monado.auxiliary.MonadoView"; + } + /*! * Wrapper for the attachToActivity static method * * Java prototype: - * `public static org.freedesktop.monado.auxiliary.MonadoView - * attachToActivity(android.app.Activity, long);` + * `public static org.freedesktop.monado.auxiliary.MonadoView attachToActivity(android.app.Activity, + * long);` * - * JNI signature: - * (Landroid/app/Activity;J)Lorg/freedesktop/monado/auxiliary/MonadoView; + * JNI signature: (Landroid/app/Activity;J)Lorg/freedesktop/monado/auxiliary/MonadoView; * */ static MonadoView attachToActivity(android::app::Activity const &activity, void *nativePointer); /*! - * Wrapper for the waitGetSurfaceHolder method + * Wrapper for the attachToActivity static method * * Java prototype: - * `public android.view.SurfaceHolder - * waitGetSurfaceHolder(int);` + * `public static org.freedesktop.monado.auxiliary.MonadoView attachToActivity(android.app.Activity);` * - * JNI signature: (I)Landroid/view/SurfaceHolder; + * JNI signature: (Landroid/app/Activity;)Lorg/freedesktop/monado/auxiliary/MonadoView; * */ - android::view::SurfaceHolder - waitGetSurfaceHolder(int32_t wait_ms); + static MonadoView + attachToActivity(android::app::Activity const &activity); + + /*! + * Wrapper for the getDisplayMetrics static method + * + * Java prototype: + * `public static android.util.DisplayMetrics getDisplayMetrics(android.app.Activity);` + * + * JNI signature: (Landroid/app/Activity;)Landroid/util/DisplayMetrics; + * + */ + static jni::Object + getDisplayMetrics(android::app::Activity const &activity); + + /*! + * Wrapper for the getNativePointer method + * + * Java prototype: + * `public long getNativePointer();` + * + * JNI signature: ()J + * + */ + void * + getNativePointer(); /*! * Wrapper for the markAsDiscardedByNative method @@ -81,9 +110,17 @@ namespace org::freedesktop::monado::auxiliary { void markAsDiscardedByNative(); - - static jni::Object - getDisplayMetrics(android::app::Activity const &activity); + /*! + * Wrapper for the waitGetSurfaceHolder method + * + * Java prototype: + * `public android.view.SurfaceHolder waitGetSurfaceHolder(int);` + * + * JNI signature: (I)Landroid/view/SurfaceHolder; + * + */ + android::view::SurfaceHolder + waitGetSurfaceHolder(int32_t wait_ms); /*! * Initialize the static metadata of this wrapper with a known @@ -101,9 +138,11 @@ namespace org::freedesktop::monado::auxiliary { struct Meta : public MetaBase { jni::method_t attachToActivity; - jni::method_t waitGetSurfaceHolder; - jni::method_t markAsDiscardedByNative; + jni::method_t attachToActivity1; jni::method_t getDisplayMetrics; + jni::method_t getNativePointer; + jni::method_t markAsDiscardedByNative; + jni::method_t waitGetSurfaceHolder; /*! * Singleton accessor @@ -116,9 +155,10 @@ namespace org::freedesktop::monado::auxiliary { } private: - Meta(jni::jclass clazz); + explicit Meta(jni::jclass clazz); }; }; + } // namespace org::freedesktop::monado::auxiliary } // namespace wrap #include "org.freedesktop.monado.auxiliary.impl.hpp" diff --git a/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.impl.hpp b/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.impl.hpp index a334933ea..a34f3fa6c 100644 --- a/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.impl.hpp +++ b/src/xrt/auxiliary/android/org.freedesktop.monado.auxiliary.impl.hpp @@ -1,4 +1,4 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -13,6 +13,7 @@ #include "wrap/android.app.h" #include "wrap/android.view.h" + namespace wrap { namespace org::freedesktop::monado::auxiliary { inline MonadoView @@ -20,7 +21,35 @@ namespace org::freedesktop::monado::auxiliary { { return MonadoView(Meta::data().clazz().call( Meta::data().attachToActivity, activity.object(), - static_cast(reinterpret_cast(nativePointer)))); + static_cast(reinterpret_cast(nativePointer)))); + } + + inline MonadoView + MonadoView::attachToActivity(android::app::Activity const &activity) + { + return MonadoView( + Meta::data().clazz().call(Meta::data().attachToActivity1, activity.object())); + } + + inline jni::Object + MonadoView::getDisplayMetrics(android::app::Activity const &activity) + { + return Meta::data().clazz().call(Meta::data().getDisplayMetrics, activity.object()); + } + + inline void * + MonadoView::getNativePointer() + { + assert(!isNull()); + return reinterpret_cast( + static_cast(object().call(Meta::data().getNativePointer))); + } + + inline void + MonadoView::markAsDiscardedByNative() + { + assert(!isNull()); + return object().call(Meta::data().markAsDiscardedByNative); } inline android::view::SurfaceHolder @@ -31,18 +60,5 @@ namespace org::freedesktop::monado::auxiliary { object().call(Meta::data().waitGetSurfaceHolder, wait_ms)); } - inline void - MonadoView::markAsDiscardedByNative() - { - assert(!isNull()); - return object().call(Meta::data().markAsDiscardedByNative); - } - - inline jni::Object - MonadoView::getDisplayMetrics(android::app::Activity const &activity) - { - return Meta::data().clazz().call(Meta::data().getDisplayMetrics, activity.object()); - } - } // namespace org::freedesktop::monado::auxiliary } // namespace wrap diff --git a/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/MonadoView.java b/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/MonadoView.java index e8dd1f28e..029b3e34a 100644 --- a/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/MonadoView.java +++ b/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/MonadoView.java @@ -10,6 +10,7 @@ package org.freedesktop.monado.auxiliary; import android.app.Activity; +import android.content.Context; import android.os.Build; import android.util.DisplayMetrics; import android.util.Log; @@ -18,6 +19,7 @@ import android.view.SurfaceView; import android.view.View; import android.view.WindowManager; +import androidx.annotation.GuardedBy; import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -30,43 +32,41 @@ import java.util.Calendar; @Keep public class MonadoView extends SurfaceView implements SurfaceHolder.Callback, SurfaceHolder.Callback2 { private static final String TAG = "MonadoView"; - @SuppressWarnings("deprecation") - private static final int sysUiVisFlags = 0 - // Give us a stable view of content insets - | View.SYSTEM_UI_FLAG_LAYOUT_STABLE - // Be able to do fullscreen and hide navigation - | View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN - | View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION - | View.SYSTEM_UI_FLAG_FULLSCREEN - | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION - // we want sticky immersive - | View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY; - /// The activity we've connected to. - private final Activity activity; - /// Guards currentSurfaceHolder + @NonNull + private final Context context; + + /// The activity we've connected to. + @Nullable + private final Activity activity; private final Object currentSurfaceHolderSync = new Object(); - private final Method viewSetSysUiVis; - private NativeCounterpart nativeCounterpart; public int width = -1; public int height = -1; public int format = -1; - /// Guarded by currentSurfaceHolderSync + private NativeCounterpart nativeCounterpart; + + @GuardedBy("currentSurfaceHolderSync") + @Nullable private SurfaceHolder currentSurfaceHolder = null; + public MonadoView(Context context) { + super(context); + this.context = context; + Activity activity; + if (context instanceof Activity) { + activity = (Activity) context; + } else { + activity = null; + } + this.activity = activity; + } + public MonadoView(Activity activity) { super(activity); + this.context = activity; this.activity = activity; - Method method; - try { - method = activity.getWindow().getDecorView().getClass().getMethod("setSystemUiVisibility", int.class); - } catch (NoSuchMethodException e) { - // ok - method = null; - } - viewSetSysUiVis = method; } private MonadoView(Activity activity, long nativePointer) { @@ -74,26 +74,11 @@ public class MonadoView extends SurfaceView implements SurfaceHolder.Callback, S nativeCounterpart = new NativeCounterpart(nativePointer); } - private void createSurface() { - Log.i(TAG, "Starting to add a new surface!"); - activity.runOnUiThread(() -> { - Log.i(TAG, "Starting runOnUiThread"); - activity.getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); - - WindowManager windowManager = activity.getWindowManager(); - windowManager.addView(this, new WindowManager.LayoutParams(WindowManager.LayoutParams.FLAG_FULLSCREEN)); - - requestFocus(); - SurfaceHolder surfaceHolder = getHolder(); - surfaceHolder.addCallback(this); - Log.i(TAG, "Registered callbacks!"); - }); - } - /** * Construct and start attaching a MonadoView to a client application. * - * @param activity The activity to attach to. + * @param activity The activity to attach to. + * @param nativePointer The native android_custom_surface pointer, cast to a long. * @return The MonadoView instance created and asynchronously attached. */ @NonNull @@ -101,7 +86,7 @@ public class MonadoView extends SurfaceView implements SurfaceHolder.Callback, S @SuppressWarnings("deprecation") public static MonadoView attachToActivity(@NonNull final Activity activity, long nativePointer) { final MonadoView view = new MonadoView(activity, nativePointer); - view.createSurface(); + view.createSurfaceInActivity(); return view; } @@ -109,10 +94,61 @@ public class MonadoView extends SurfaceView implements SurfaceHolder.Callback, S @Keep public static MonadoView attachToActivity(@NonNull final Activity activity) { final MonadoView view = new MonadoView(activity); - view.createSurface(); + view.createSurfaceInActivity(); return view; } + @NonNull + @Keep + public static DisplayMetrics getDisplayMetrics(Activity activity) { + DisplayMetrics displayMetrics = new DisplayMetrics(); + activity.getWindowManager().getDefaultDisplay().getMetrics(displayMetrics); + return displayMetrics; + } + + @Keep + public long getNativePointer() { + if (nativeCounterpart == null) { + return 0; + } + return nativeCounterpart.getNativePointer(); + } + + private void createSurfaceInActivity() { + createSurfaceInActivity(false); + } + + /** + * @param focusable Indicates MonadoView should be focusable or not + */ + private void createSurfaceInActivity(boolean focusable) { + Log.i(TAG, "Starting to add a new surface!"); + activity.runOnUiThread(() -> { + Log.i(TAG, "Starting runOnUiThread"); + activity.getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + + WindowManager windowManager = activity.getWindowManager(); + WindowManager.LayoutParams lp = new WindowManager.LayoutParams(); + if (focusable) { + lp.flags = WindowManager.LayoutParams.FLAG_FULLSCREEN; + } else { + // There are 2 problems if view is focusable on all-in-one device: + // 1. Navigation bar won't go away because view gets focus. + // 2. Underlying activity lost focus and can not receive input. + lp.flags = WindowManager.LayoutParams.FLAG_FULLSCREEN | + WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE | + WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE; + } + windowManager.addView(this, lp); + if (focusable) { + requestFocus(); + } + SurfaceHolder surfaceHolder = getHolder(); + surfaceHolder.addCallback(this); + Log.i(TAG, "Registered callbacks!"); + }); + } + /** * Block up to a specified amount of time, waiting for the surfaceCreated callback to be fired * and populate the currentSurfaceHolder. @@ -162,41 +198,6 @@ public class MonadoView extends SurfaceView implements SurfaceHolder.Callback, S nativeCounterpart.markAsDiscardedByNative(TAG); } - private boolean makeFullscreen() { - if (activity == null) { - return false; - } - if (viewSetSysUiVis == null) { - return false; - } - View decorView = activity.getWindow().getDecorView(); - //! @todo implement with WindowInsetsController to ward off the stink of deprecation - try { - viewSetSysUiVis.invoke(decorView, sysUiVisFlags); - } catch (IllegalAccessException e) { - return false; - } catch (InvocationTargetException e) { - return false; - } - return true; - } - - - /** - * Add a listener so that if our system UI display state doesn't include all we want, we re-apply. - */ - @RequiresApi(api = Build.VERSION_CODES.HONEYCOMB) - @SuppressWarnings("deprecation") - private void setSystemUiVisChangeListener() { - activity.getWindow().getDecorView().setOnSystemUiVisibilityChangeListener(visibility -> { - // If not fullscreen, fix it. - if (0 == (visibility & View.SYSTEM_UI_FLAG_FULLSCREEN)) { - makeFullscreen(); - } - }); - - } - @Override public void surfaceCreated(@NonNull SurfaceHolder surfaceHolder) { synchronized (currentSurfaceHolderSync) { @@ -204,11 +205,6 @@ public class MonadoView extends SurfaceView implements SurfaceHolder.Callback, S currentSurfaceHolderSync.notifyAll(); } Log.i(TAG, "surfaceCreated: Got a surface holder!"); - - if (makeFullscreen()) { - // If we could make it full screen, make it really stick. - setSystemUiVisChangeListener(); - } } @Override @@ -249,12 +245,4 @@ public class MonadoView extends SurfaceView implements SurfaceHolder.Callback, S Log.i(TAG, "surfaceRedrawNeeded"); } - @NonNull - @Keep - public static DisplayMetrics getDisplayMetrics(Activity activity) { - DisplayMetrics displayMetrics = new DisplayMetrics(); - activity.getWindowManager().getDefaultDisplay().getMetrics(displayMetrics); - return displayMetrics; - } - } diff --git a/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/SystemUiController.kt b/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/SystemUiController.kt new file mode 100644 index 000000000..eb1388e75 --- /dev/null +++ b/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/SystemUiController.kt @@ -0,0 +1,95 @@ +// Copyright 2021, Qualcomm Innovation Center, Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Class to handle system ui visibility + * @author Jarvis Huang + * @ingroup aux_android_java + */ +package org.freedesktop.monado.auxiliary + +import android.app.Activity +import android.os.Build +import android.view.View +import android.view.WindowInsets +import android.view.WindowInsetsController +import androidx.annotation.RequiresApi + +/** + * Helper class that handles system ui visibility. + */ +class SystemUiController(activity: Activity) { + private val impl: Impl = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + WindowInsetsControllerImpl(activity) + } else { + SystemUiVisibilityImpl(activity) + } + + /** + * Hide system ui and make fullscreen. + */ + fun hide() { + impl.hide() + } + + private abstract class Impl(var activity: Activity) { + abstract fun hide() + fun runOnUiThread(runnable: Runnable) { + activity.runOnUiThread(runnable) + } + } + + @Suppress("DEPRECATION") + private class SystemUiVisibilityImpl(activity: Activity) : Impl(activity) { + override fun hide() { + activity.runOnUiThread { + activity.window.decorView.systemUiVisibility = FLAG_FULL_SCREEN_IMMERSIVE_STICKY + } + } + + companion object { + private const val FLAG_FULL_SCREEN_IMMERSIVE_STICKY = + // Give us a stable view of content insets + (View.SYSTEM_UI_FLAG_LAYOUT_STABLE // Be able to do fullscreen and hide navigation + or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN + or View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION + or View.SYSTEM_UI_FLAG_FULLSCREEN + or View.SYSTEM_UI_FLAG_HIDE_NAVIGATION // we want sticky immersive + or View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY) + } + + init { + runOnUiThread { + activity.window.decorView.setOnSystemUiVisibilityChangeListener { visibility: Int -> + // If not fullscreen, fix it. + if (0 == visibility and View.SYSTEM_UI_FLAG_FULLSCREEN) { + hide() + } + } + } + } + } + + @RequiresApi(api = Build.VERSION_CODES.R) + private class WindowInsetsControllerImpl(activity: Activity) : Impl(activity) { + override fun hide() { + activity.runOnUiThread { + val controller = activity.window.insetsController + controller!!.hide(WindowInsets.Type.statusBars() or WindowInsets.Type.navigationBars()) + controller.systemBarsBehavior = + WindowInsetsController.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE + } + } + + init { + runOnUiThread { + activity.window.insetsController!!.addOnControllableInsetsChangedListener { _: WindowInsetsController?, typeMask: Int -> + if (typeMask and WindowInsets.Type.statusBars() == 1 || typeMask and WindowInsets.Type.navigationBars() == 1) { + hide() + } + } + } + } + } + +} diff --git a/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/UiProvider.kt b/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/UiProvider.kt index 3a9c89c7e..b46f3341a 100644 --- a/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/UiProvider.kt +++ b/src/xrt/auxiliary/android/src/main/java/org/freedesktop/monado/auxiliary/UiProvider.kt @@ -9,8 +9,6 @@ package org.freedesktop.monado.auxiliary import android.app.PendingIntent -import android.content.Context -import android.graphics.drawable.Drawable import android.graphics.drawable.Icon /** diff --git a/src/xrt/auxiliary/bindings/bindings.json b/src/xrt/auxiliary/bindings/bindings.json index 22e3bf1d7..22abbf482 100644 --- a/src/xrt/auxiliary/bindings/bindings.json +++ b/src/xrt/auxiliary/bindings/bindings.json @@ -450,7 +450,7 @@ "localized_name": "Right Haptic", "features": ["haptic"], "monado_bindings": { - "haptic": "XRT_OUTPUT_NAME_XBOX_HAPTIC_RIGHTT" + "haptic": "XRT_OUTPUT_NAME_XBOX_HAPTIC_RIGHT" } }, "/output/haptic_left_trigger": { diff --git a/src/xrt/auxiliary/bindings/bindings.py b/src/xrt/auxiliary/bindings/bindings.py index 8366a31d7..73207daa9 100755 --- a/src/xrt/auxiliary/bindings/bindings.py +++ b/src/xrt/auxiliary/bindings/bindings.py @@ -4,8 +4,8 @@ """Generate code from a JSON file describing interaction profiles and bindings.""" -import json import argparse +import json def handle_subpath(pathgroup_cls, feature_list, subaction_path, sub_path_itm): @@ -31,7 +31,8 @@ class Feature: feature_list = [] for subaction_path in subaction_paths: for sub_path_itm in paths.items(): - handle_subpath(feature_cls, feature_list, subaction_path, sub_path_itm) + handle_subpath(feature_cls, feature_list, + subaction_path, sub_path_itm) return feature_list def __init__(self, subaction_path, sub_path_itm, feature_str): @@ -41,10 +42,10 @@ class Feature: self.subaction_path = subaction_path self.feature_str = feature_str - """A group of paths that derive from the same input. - For example .../thumbstick, .../thumbstick/x, .../thumbstick/y - """ def to_monado_paths(self): + """A group of paths that derive from the same input. + For example .../thumbstick, .../thumbstick/x, .../thumbstick/y + """ paths = [] basepath = self.subaction_path + self.sub_path_name @@ -60,7 +61,7 @@ class Feature: return paths def is_input(self): - # only haptics is output so far, everythine else is input + # only haptics is output so far, everything else is input return self.feature_str != "haptic" def is_output(self): @@ -68,7 +69,8 @@ class Feature: class Profile: - """An interctive bindings profile.""" + """An interactive bindings profile.""" + def __init__(self, name, data): """Construct an profile.""" self.name = name @@ -83,7 +85,7 @@ class Profile: for feature in self.features: for path in feature.to_monado_paths(): length = len(path) - if (length in self.by_length): + if length in self.by_length: self.by_length[length].append(path) else: self.by_length[length] = [path] @@ -152,7 +154,8 @@ def generate_bindings_c(file, p): f.write("{\n\t\t\treturn false;\n\t\t}\n") f.write("\tdefault:\n\t\treturn false;\n\t}\n}\n") - f.write(f'\n\nstruct profile_template profile_templates[{len(p.profiles)}] = {{ // array of profile_template\n') + f.write( + f'\n\nstruct profile_template profile_templates[{len(p.profiles)}] = {{ // array of profile_template\n') for profile in p.profiles: hw_name = str(profile.name.split("/")[-1]) vendor_name = str(profile.name.split("/")[-2]) @@ -167,7 +170,8 @@ def generate_bindings_c(file, p): f.write(f'\t\t.steamvr_input_profile_path = "{fname}",\n') f.write(f'\t\t.steamvr_controller_type = "{controller_type}",\n') f.write(f'\t\t.num_bindings = {num_bindings},\n') - f.write(f'\t\t.bindings = (struct binding_template[]){{ // array of binding_template\n') + f.write( + f'\t\t.bindings = (struct binding_template[]){{ // array of binding_template\n') feature: Feature for idx, feature in enumerate(profile.features): @@ -180,7 +184,8 @@ def generate_bindings_c(file, p): f.write(f'\t\t\t{{ // binding_template {idx}\n') f.write(f'\t\t\t\t.subaction_path = "{feature.subaction_path}",\n') f.write(f'\t\t\t\t.steamvr_path = "{steamvr_path}",\n') - f.write(f'\t\t\t\t.localized_name = "{sp_obj["localized_name"]}",\n') + f.write( + f'\t\t\t\t.localized_name = "{sp_obj["localized_name"]}",\n') f.write('\t\t\t\t.paths = { // array of paths\n') for path in feature.to_monado_paths(): @@ -215,6 +220,42 @@ def generate_bindings_c(file, p): f.write('}; // /array of profile_template\n\n') + inputs = set() + for profile in p.profiles: + feature: Feature + for idx, feature in enumerate(profile.features): + sp_obj = feature.sub_path_obj + if feature_str not in sp_obj["monado_bindings"]: + continue + monado_binding = sp_obj["monado_bindings"][feature_str] + inputs.add(monado_binding) + + # special cased bindings that are never directly used in the input profiles + inputs.add("XRT_INPUT_GENERIC_HEAD_POSE") + inputs.add("XRT_INPUT_GENERIC_HEAD_DETECT") + inputs.add("XRT_INPUT_GENERIC_HAND_TRACKING_LEFT") + inputs.add("XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT") + inputs.add("XRT_INPUT_GENERIC_TRACKER_POSE") + + f.write('const char *\n') + f.write('xrt_input_name_string(enum xrt_input_name input)\n') + f.write('{\n') + f.write('\tswitch(input)\n') + f.write('\t{\n') + for input in inputs: + f.write(f'\tcase {input}: return "{input}";\n') + f.write(f'\tdefault: return "UNKNOWN";\n') + f.write('\t}\n') + f.write('}\n') + + f.write('enum xrt_input_name\n') + f.write('xrt_input_name_enum(const char *input)\n') + f.write('{\n') + for input in inputs: + f.write(f'\tif(strcmp("{input}", input) == 0) return {input};\n') + f.write(f'\treturn XRT_INPUT_GENERIC_TRACKER_POSE;\n') + f.write('}\n') + f.write("\n// clang-format on\n") f.close() @@ -240,13 +281,14 @@ def generate_bindings_h(file, p): "_subpath(const char *str, size_t length);\n") f.write(f''' +#define PATHS_PER_BINDING_TEMPLATE 8 struct binding_template {{ \tconst char *subaction_path; \tconst char *steamvr_path; \tconst char *localized_name; -\tconst char *paths[8]; +\tconst char *paths[PATHS_PER_BINDING_TEMPLATE]; \tenum xrt_input_name input; \tenum xrt_output_name output; }}; @@ -263,9 +305,16 @@ struct profile_template }}; #define NUM_PROFILE_TEMPLATES {len(p.profiles)} -extern struct profile_template profile_templates[{len(p.profiles)}]; +extern struct profile_template profile_templates[NUM_PROFILE_TEMPLATES]; + ''') + f.write('const char *\n') + f.write('xrt_input_name_string(enum xrt_input_name input);\n\n') + + f.write('enum xrt_input_name\n') + f.write('xrt_input_name_enum(const char *input);\n\n') + f.write("\n// clang-format on\n") f.close() diff --git a/src/xrt/auxiliary/bindings/meson.build b/src/xrt/auxiliary/bindings/meson.build index 2cb8c20b1..af3e54566 100644 --- a/src/xrt/auxiliary/bindings/meson.build +++ b/src/xrt/auxiliary/bindings/meson.build @@ -35,5 +35,6 @@ lib_aux_generated_bindings = static_library( aux_generated_bindings = declare_dependency( include_directories: aux_include, + sources: [generated_bindings_h], link_with: lib_aux_generated_bindings, ) diff --git a/src/xrt/auxiliary/gstreamer/gst_internal.h b/src/xrt/auxiliary/gstreamer/gst_internal.h new file mode 100644 index 000000000..d975bc8af --- /dev/null +++ b/src/xrt/auxiliary/gstreamer/gst_internal.h @@ -0,0 +1,80 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Semi internal structs for gstreamer code. + * @author Jakob Bornecrantz + * @ingroup aux_util + */ + +#pragma once + +#include "xrt/xrt_frame.h" + +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* + * + * Pipeline + * + */ + +/*! + * A pipeline from which you can create one or more @ref gstreamer_sink from. + * + * @implements xrt_frame_node + */ +struct gstreamer_pipeline +{ + struct xrt_frame_node node; + + struct xrt_frame_context *xfctx; + + GstElement *pipeline; +}; + + +/* + * + * Sink + * + */ + +/*! + * An @ref xrt_frame_sink that uses appsrc. + * + * @implements xrt_frame_sink + * @implements xrt_frame_node + */ +struct gstreamer_sink +{ + //! The base structure exposing the sink interface. + struct xrt_frame_sink base; + + //! A sink can expose multie @ref xrt_frame_sink but only one node. + struct xrt_frame_node node; + + //! Pipeline this sink is producing frames into. + struct gstreamer_pipeline *gp; + + //! Offset applied to timestamps given to GStreamer. + uint64_t offset_ns; + + //! Last sent timestamp, used to calculate duration. + uint64_t timestamp_ns; + + //! Cached appsrc element. + GstElement *appsrc; +}; + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/gstreamer/gst_pipeline.c b/src/xrt/auxiliary/gstreamer/gst_pipeline.c new file mode 100644 index 000000000..43fb619f8 --- /dev/null +++ b/src/xrt/auxiliary/gstreamer/gst_pipeline.c @@ -0,0 +1,152 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief An @ref xrt_frame_sink that does gst things. + * @author Jakob Bornecrantz + * @author Aaron Boxer + * @ingroup aux_util + */ + +#include "util/u_misc.h" +#include "util/u_debug.h" + +#include "gstreamer/gst_internal.h" +#include "gstreamer/gst_pipeline.h" + + +/* + * + * Internal pipeline functions. + * + */ + +static void +break_apart(struct xrt_frame_node *node) +{ + struct gstreamer_pipeline *gp = container_of(node, struct gstreamer_pipeline, node); + + /* + * This function is called when we are shutting down, after returning + * from this function you are not allowed to call any other nodes in the + * graph. But it must be safe for other nodes to call any normal + * functions on us. Once the context is done calling break_aprt on all + * objects it will call destroy on them. + */ + + (void)gp; +} + +static void +destroy(struct xrt_frame_node *node) +{ + struct gstreamer_pipeline *gp = container_of(node, struct gstreamer_pipeline, node); + + /* + * All of the nodes has been broken apart and none of our functions will + * be called, it's now safe to destroy and free ourselves. + */ + + free(gp); +} + + +/* + * + * Exported functions. + * + */ + +void +gstreamer_pipeline_play(struct gstreamer_pipeline *gp) +{ + U_LOG_D("Starting pipeline"); + + gst_element_set_state(gp->pipeline, GST_STATE_PLAYING); +} + +void +gstreamer_pipeline_stop(struct gstreamer_pipeline *gp) +{ + U_LOG_D("Stopping pipeline"); + + // Settle the pipeline. + U_LOG_T("Sending EOS"); + gst_element_send_event(gp->pipeline, gst_event_new_eos()); + + // Wait for EOS message on the pipeline bus. + U_LOG_T("Waiting for EOS"); + GstMessage *msg = NULL; + msg = gst_bus_timed_pop_filtered(GST_ELEMENT_BUS(gp->pipeline), GST_CLOCK_TIME_NONE, + GST_MESSAGE_EOS | GST_MESSAGE_ERROR); + //! @todo Should check if we got an error message here or an eos. + (void)msg; + + // Completely stop the pipeline. + U_LOG_T("Setting to NULL"); + gst_element_set_state(gp->pipeline, GST_STATE_NULL); +} + +void +gstreamer_pipeline_create_from_string(struct xrt_frame_context *xfctx, + const char *pipeline_string, + struct gstreamer_pipeline **out_gp) +{ + gst_init(NULL, NULL); + + struct gstreamer_pipeline *gp = U_TYPED_CALLOC(struct gstreamer_pipeline); + gp->node.break_apart = break_apart; + gp->node.destroy = destroy; + gp->xfctx = xfctx; + + // Setup pipeline. + gp->pipeline = gst_parse_launch(pipeline_string, NULL); + + /* + * Add ourselves to the context so we are destroyed. + * This is done once we know everything is completed. + */ + xrt_frame_context_add(xfctx, &gp->node); + + *out_gp = gp; +} + +void +gstreamer_pipeline_create_autovideo_sink(struct xrt_frame_context *xfctx, + const char *appsrc_name, + struct gstreamer_pipeline **out_gp) +{ + gst_init(NULL, NULL); + + struct gstreamer_pipeline *gp = U_TYPED_CALLOC(struct gstreamer_pipeline); + gp->node.break_apart = break_apart; + gp->node.destroy = destroy; + gp->xfctx = xfctx; + + // Setup pipeline. + gp->pipeline = gst_pipeline_new("pipeline"); + GstElement *appsrc = gst_element_factory_make("appsrc", appsrc_name); + GstElement *conv = gst_element_factory_make("videoconvert", "conv"); + GstElement *scale = gst_element_factory_make("videoscale", "scale"); + GstElement *videosink = gst_element_factory_make("autovideosink", "videosink"); + + gst_bin_add_many(GST_BIN(gp->pipeline), // + appsrc, // + conv, // + scale, // + videosink, // + NULL); + gst_element_link_many(appsrc, // + conv, // + scale, // + videosink, // + NULL); + + /* + * Add ourselves to the context so we are destroyed. + * This is done once we know everything is completed. + */ + xrt_frame_context_add(xfctx, &gp->node); + + *out_gp = gp; +} diff --git a/src/xrt/auxiliary/gstreamer/gst_pipeline.h b/src/xrt/auxiliary/gstreamer/gst_pipeline.h new file mode 100644 index 000000000..c98c279c9 --- /dev/null +++ b/src/xrt/auxiliary/gstreamer/gst_pipeline.h @@ -0,0 +1,40 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Functions for creating @ref gstreamer_pipeline objects. + * @author Jakob Bornecrantz + * @ingroup aux_util + */ + +#pragma once + +#include "xrt/xrt_frame.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +struct gstreamer_pipeline; + +void +gstreamer_pipeline_create_from_string(struct xrt_frame_context *xfctx, + const char *pipeline_string, + struct gstreamer_pipeline **out_gp); + +void +gstreamer_pipeline_create_autovideo_sink(struct xrt_frame_context *xfctx, + const char *appsrc_name, + struct gstreamer_pipeline **out_gp); + +void +gstreamer_pipeline_play(struct gstreamer_pipeline *gp); + +void +gstreamer_pipeline_stop(struct gstreamer_pipeline *gp); + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/gstreamer/gst_sink.c b/src/xrt/auxiliary/gstreamer/gst_sink.c new file mode 100644 index 000000000..33f897bfa --- /dev/null +++ b/src/xrt/auxiliary/gstreamer/gst_sink.c @@ -0,0 +1,193 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief An @ref xrt_frame_sink that does gst things. + * @author Jakob Bornecrantz + * @author Aaron Boxer + * @ingroup aux_util + */ + +#include "util/u_misc.h" +#include "util/u_debug.h" +#include "util/u_format.h" + +#include "gstreamer/gst_sink.h" +#include "gstreamer/gst_pipeline.h" +#include "gstreamer/gst_internal.h" + +#include + + +/* + * + * Internal sink functions. + * + */ + +static void +wrapped_buffer_destroy(gpointer data) +{ + struct xrt_frame *xf = (struct xrt_frame *)data; + + U_LOG_T("Called"); + + xrt_frame_reference(&xf, NULL); +} + +static void +push_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) +{ + struct gstreamer_sink *gs = (struct gstreamer_sink *)xfs; + GstBuffer *buffer; + GstFlowReturn ret; + + U_LOG_T( + "Called" + "\n\tformat: %s" + "\n\twidth: %u" + "\n\theight: %u", + u_format_str(xf->format), xf->width, xf->height); + + /* We need to take a reference on the frame to keep it alive. */ + struct xrt_frame *taken = NULL; + xrt_frame_reference(&taken, xf); + + /* Wrap the frame that we now hold a reference to. */ + buffer = gst_buffer_new_wrapped_full( // + 0, // GstMemoryFlags flags + (gpointer)xf->data, // gpointer data + taken->size, // gsize maxsize + 0, // gsize offset + taken->size, // gsize size + taken, // gpointer user_data + wrapped_buffer_destroy); // GDestroyNotify notify + + //! Get the timestampe from the frame. + uint64_t xtimestamp_ns = xf->timestamp; + + // Use the first frame as offset. + if (gs->offset_ns == 0) { + gs->offset_ns = xtimestamp_ns; + } + + // Need to be offset or gstreamer becomes sad. + GST_BUFFER_PTS(buffer) = xtimestamp_ns - gs->offset_ns; + + // Duration is measured from last time stamp. + GST_BUFFER_DURATION(buffer) = xtimestamp_ns - gs->timestamp_ns; + gs->timestamp_ns = xtimestamp_ns; + + // All done, send it to the gstreamer pipeline. + ret = gst_app_src_push_buffer((GstAppSrc *)gs->appsrc, buffer); + if (ret != GST_FLOW_OK) { + U_LOG_E("Got GST error '%i'", ret); + } +} + +static void +enough_data(GstElement *appsrc, gpointer udata) +{ + // Debugging code. + U_LOG_T("Called"); +} + +static void +break_apart(struct xrt_frame_node *node) +{ + struct gstreamer_sink *gs = container_of(node, struct gstreamer_sink, node); + + /* + * This function is called when we are shutting down, after returning + * from this function you are not allowed to call any other nodes in the + * graph. But it must be safe for other nodes to call any normal + * functions on us. Once the context is done calling break_aprt on all + * objects it will call destroy on them. + */ + + (void)gs; +} + +static void +destroy(struct xrt_frame_node *node) +{ + struct gstreamer_sink *gs = container_of(node, struct gstreamer_sink, node); + + /* + * All of the nodes has been broken apart and none of our functions will + * be called, it's now safe to destroy and free ourselves. + */ + + free(gs); +} + + +/* + * + * Exported functions. + * + */ + +void +gstreamer_sink_send_eos(struct gstreamer_sink *gs) +{ + gst_element_send_event(gs->appsrc, gst_event_new_eos()); +} + +uint64_t +gstreamer_sink_get_timestamp_offset(struct gstreamer_sink *gs) +{ + return gs->offset_ns; +} + +void +gstreamer_sink_create_with_pipeline(struct gstreamer_pipeline *gp, + uint32_t width, + uint32_t height, + enum xrt_format format, + const char *appsrc_name, + struct gstreamer_sink **out_gs, + struct xrt_frame_sink **out_xfs) +{ + const char *format_str = NULL; + switch (format) { + case XRT_FORMAT_R8G8B8: format_str = "RGB"; break; + case XRT_FORMAT_YUYV422: format_str = "YUY2"; break; + case XRT_FORMAT_L8: format_str = "GRAY8"; break; + default: assert(false); break; + } + + struct gstreamer_sink *gs = U_TYPED_CALLOC(struct gstreamer_sink); + gs->base.push_frame = push_frame; + gs->node.break_apart = break_apart; + gs->node.destroy = destroy; + gs->gp = gp; + gs->appsrc = gst_bin_get_by_name(GST_BIN(gp->pipeline), appsrc_name); + + + GstCaps *caps = gst_caps_new_simple( // + "video/x-raw", // + "format", G_TYPE_STRING, format_str, // + "width", G_TYPE_INT, width, // + "height", G_TYPE_INT, height, // + "framerate", GST_TYPE_FRACTION, 0, 1, // + NULL); + + g_object_set(G_OBJECT(gs->appsrc), // + "caps", caps, // + "stream-type", GST_APP_STREAM_TYPE_STREAM, // + "format", GST_FORMAT_TIME, // + "is-live", TRUE, // + NULL); + + g_signal_connect(G_OBJECT(gs->appsrc), "enough-data", G_CALLBACK(enough_data), gs); + + /* + * Add ourselves to the context so we are destroyed. + * This is done once we know everything is completed. + */ + xrt_frame_context_add(gp->xfctx, &gs->node); + + *out_gs = gs; + *out_xfs = &gs->base; +} diff --git a/src/xrt/auxiliary/gstreamer/gst_sink.h b/src/xrt/auxiliary/gstreamer/gst_sink.h new file mode 100644 index 000000000..c9917d65a --- /dev/null +++ b/src/xrt/auxiliary/gstreamer/gst_sink.h @@ -0,0 +1,40 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief @ref xrt_frame_sink that does gst things. + * @author Jakob Bornecrantz + * @ingroup aux_util + */ + +#pragma once + +#include "xrt/xrt_frame.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct gstreamer_sink; +struct gstreamer_pipeline; + + +void +gstreamer_sink_send_eos(struct gstreamer_sink *gp); + +uint64_t +gstreamer_sink_get_timestamp_offset(struct gstreamer_sink *gs); + +void +gstreamer_sink_create_with_pipeline(struct gstreamer_pipeline *gp, + uint32_t width, + uint32_t height, + enum xrt_format format, + const char *appsrc_name, + struct gstreamer_sink **out_gs, + struct xrt_frame_sink **out_xfs); + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/math/m_api.h b/src/xrt/auxiliary/math/m_api.h index 2369e6d83..65a4c1b3a 100644 --- a/src/xrt/auxiliary/math/m_api.h +++ b/src/xrt/auxiliary/math/m_api.h @@ -1,9 +1,10 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief C interface to math library. * @author Jakob Bornecrantz + * @author Moses Turner * * @see xrt_vec3 * @see xrt_quat @@ -147,7 +148,7 @@ math_vec3_normalize(struct xrt_vec3 *in); * Create a rotation from a angle in radians and a vector. * * @relates xrt_quat - * @relates xrt_vec3 + * @see xrt_vec3 * @ingroup aux_math */ void @@ -157,7 +158,7 @@ math_quat_from_angle_vector(float angle_rads, const struct xrt_vec3 *vector, str * Create a rotation from a 3x3 rotation matrix. * * @relates xrt_quat - * @relates xrt_matrix_3x3 + * @see xrt_matrix_3x3 * @ingroup aux_math */ void @@ -168,7 +169,7 @@ math_quat_from_matrix_3x3(const struct xrt_matrix_3x3 *mat, struct xrt_quat *res * matrix by crossing z and x to get the y axis. * * @relates xrt_quat - * @relates xrt_vec3 + * @see xrt_vec3 * @ingroup aux_math */ void @@ -226,7 +227,7 @@ math_quat_ensure_normalized(struct xrt_quat *inout); * Rotate a vector. * * @relates xrt_quat - * @relatesalso xrt_vec3 + * @see xrt_vec3 * @ingroup aux_math */ void @@ -250,7 +251,7 @@ math_quat_rotate(const struct xrt_quat *left, const struct xrt_quat *right, stru * vector should be in radians per unit of time. * * @relates xrt_quat - * @relatesalso xrt_vec3 + * @see xrt_vec3 * @ingroup aux_math */ void @@ -269,7 +270,7 @@ math_quat_integrate_velocity(const struct xrt_quat *quat, * radians per unit of time. * * @relates xrt_quat - * @relatesalso xrt_vec3 + * @see xrt_vec3 * @ingroup aux_math */ void @@ -282,13 +283,22 @@ math_quat_finite_difference(const struct xrt_quat *quat0, * Used to rotate a derivative like a angular velocity. * * @relates xrt_quat - * @relatesalso xrt_vec3 + * @see xrt_vec3 * @ingroup aux_math */ void math_quat_rotate_derivative(const struct xrt_quat *rot, const struct xrt_vec3 *deriv, struct xrt_vec3 *result); +/*! + * Slerp (spherical linear interpolation) between two quaternions + * + * @relates xrt_quat + * @ingroup aux_math + */ +void +math_quat_slerp(const struct xrt_quat *left, const struct xrt_quat *right, float t, struct xrt_quat *result); + /* * * Matrix functions @@ -311,6 +321,26 @@ math_matrix_3x3_transform_vec3(const struct xrt_matrix_3x3 *left, const struct xrt_vec3 *right, struct xrt_vec3 *result); +/*! + * Multiply Matrix3x3. + * + * @relates xrt_matrix_3x3 + * @ingroup aux_math + */ +void +math_matrix_3x3_multiply(const struct xrt_matrix_3x3 *left, + const struct xrt_matrix_3x3 *right, + struct xrt_matrix_3x3 *result); + +/*! + * Invert Matrix3x3 + * + * @relates xrt_matrix_3x3 + * @ingroup aux_math + */ +void +math_matrix_3x3_inverse(const struct xrt_matrix_3x3 *in, struct xrt_matrix_3x3 *result); + /*! * Initialize Matrix4x4 with identity. * @@ -367,6 +397,16 @@ math_matrix_4x4_inverse_view_projection(const struct xrt_matrix_4x4 *view, * */ + +/*! + * Somewhat laboriously make an xrt_pose identity. + * + * @relates xrt_pose + * @ingroup aux_math + */ +void +math_pose_identity(struct xrt_pose *pose); + /*! * Check if this pose can be used in transformation operations. * @@ -404,13 +444,35 @@ math_pose_transform(const struct xrt_pose *transform, const struct xrt_pose *pos * The input point and output may be the same pointer. * * @relates xrt_pose - * @relates xrt_vec3 + * @see xrt_vec3 * @ingroup aux_math */ void math_pose_transform_point(const struct xrt_pose *transform, const struct xrt_vec3 *point, struct xrt_vec3 *out_point); +/* + * + * Inline functions. + * + */ + +/*! + * Map a number from one range to another range. + * Exactly the same as Arduino's map(). + */ +static inline double +math_map_ranges(double value, double from_low, double from_high, double to_low, double to_high) +{ + return (value - from_low) * (to_high - to_low) / (from_high - from_low) + to_low; +} + +static inline double +math_lerp(double from, double to, double amount) +{ + return (from * (1.0 - amount)) + (to * (amount)); +} + /* * * Optics functions. diff --git a/src/xrt/auxiliary/math/m_base.cpp b/src/xrt/auxiliary/math/m_base.cpp index b01e68064..ff0952aad 100644 --- a/src/xrt/auxiliary/math/m_base.cpp +++ b/src/xrt/auxiliary/math/m_base.cpp @@ -5,6 +5,7 @@ * @brief Base implementations for math library. * @author Jakob Bornecrantz * @author Ryan Pavlik + * @author Moses Turner * @ingroup aux_math */ @@ -16,6 +17,7 @@ #include +using namespace xrt::auxiliary::math; /* * @@ -38,6 +40,21 @@ copy(const struct xrt_quat *q) return copy(*q); } +XRT_MAYBE_UNUSED static inline Eigen::Quaterniond +copyd(const struct xrt_quat &q) +{ + // Eigen constructor order is different from XRT, OpenHMD and OpenXR! + // Eigen: `float w, x, y, z`. + // OpenXR: `float x, y, z, w`. + return Eigen::Quaterniond(q.w, q.x, q.y, q.z); +} + +XRT_MAYBE_UNUSED static inline Eigen::Quaterniond +copyd(const struct xrt_quat *q) +{ + return copyd(*q); +} + static inline Eigen::Vector3f copy(const struct xrt_vec3 &v) { @@ -50,6 +67,30 @@ copy(const struct xrt_vec3 *v) return copy(*v); } +XRT_MAYBE_UNUSED static inline Eigen::Vector3d +copyd(const struct xrt_vec3 &v) +{ + return Eigen::Vector3d(v.x, v.y, v.z); +} + +XRT_MAYBE_UNUSED static inline Eigen::Vector3d +copyd(const struct xrt_vec3 *v) +{ + return copyd(*v); +} + +static inline Eigen::Matrix3f +copy(const struct xrt_matrix_3x3 *m) +{ + Eigen::Matrix3f res; + // clang-format off + res << m->v[0], m->v[3], m->v[6], + m->v[1], m->v[4], m->v[7], + m->v[2], m->v[5], m->v[8]; + // clang-format on + return res; +} + static inline Eigen::Matrix4f copy(const struct xrt_matrix_4x4 *m) { @@ -63,6 +104,7 @@ copy(const struct xrt_matrix_4x4 *m) return res; } + /* * * Exported vector functions. @@ -200,7 +242,7 @@ math_quat_validate(const struct xrt_quat *quat) extern "C" bool math_quat_validate_within_1_percent(const struct xrt_quat *quat) { - return quat_validate(0.01, quat); + return quat_validate(0.01f, quat); } extern "C" void @@ -285,6 +327,18 @@ math_quat_rotate_derivative(const struct xrt_quat *quat, const struct xrt_vec3 * *result = ret; } +extern "C" void +math_quat_slerp(const struct xrt_quat *left, const struct xrt_quat *right, float t, struct xrt_quat *result) +{ + assert(left != NULL); + assert(right != NULL); + assert(result != NULL); + + auto l = copy(left); + auto r = copy(right); + + map_quat(*result) = l.slerp(t, r); +} /* * @@ -314,6 +368,30 @@ math_matrix_3x3_transform_vec3(const struct xrt_matrix_3x3 *left, const struct x map_vec3(*result) = m * copy(right); } +extern "C" void +math_matrix_3x3_multiply(const struct xrt_matrix_3x3 *left, + const struct xrt_matrix_3x3 *right, + struct xrt_matrix_3x3 *result) +{ + result->v[0] = left->v[0] * right->v[0] + left->v[1] * right->v[3] + left->v[2] * right->v[6]; + result->v[1] = left->v[0] * right->v[1] + left->v[1] * right->v[4] + left->v[2] * right->v[7]; + result->v[2] = left->v[0] * right->v[2] + left->v[1] * right->v[5] + left->v[2] * right->v[8]; + + result->v[3] = left->v[3] * right->v[0] + left->v[4] * right->v[3] + left->v[5] * right->v[6]; + result->v[4] = left->v[3] * right->v[1] + left->v[4] * right->v[4] + left->v[5] * right->v[7]; + result->v[5] = left->v[3] * right->v[2] + left->v[4] * right->v[5] + left->v[5] * right->v[8]; + + result->v[6] = left->v[6] * right->v[0] + left->v[7] * right->v[3] + left->v[8] * right->v[6]; + result->v[7] = left->v[6] * right->v[1] + left->v[7] * right->v[4] + left->v[8] * right->v[7]; + result->v[8] = left->v[6] * right->v[2] + left->v[7] * right->v[5] + left->v[8] * right->v[8]; +} + +extern "C" void +math_matrix_3x3_inverse(const struct xrt_matrix_3x3 *in, struct xrt_matrix_3x3 *result) +{ + Eigen::Matrix3f m = copy(in); + map_matrix_3x3(*result) = m.inverse(); +} void math_matrix_4x4_identity(struct xrt_matrix_4x4 *result) @@ -367,6 +445,70 @@ math_matrix_4x4_inverse_view_projection(const struct xrt_matrix_4x4 *view, map_matrix_4x4(*result) = vp.inverse(); } + +/* + * + * Exported Matrix 4x4 functions. + * + */ + +extern "C" void +m_mat4_f64_identity(struct xrt_matrix_4x4_f64 *result) +{ + map_matrix_4x4_f64(*result) = Eigen::Matrix4d::Identity(); +} + +extern "C" void +m_mat4_f64_invert(const struct xrt_matrix_4x4_f64 *matrix, struct xrt_matrix_4x4_f64 *result) +{ + Eigen::Matrix4d m = map_matrix_4x4_f64(*matrix); + map_matrix_4x4_f64(*result) = m.inverse(); +} + +extern "C" void +m_mat4_f64_multiply(const struct xrt_matrix_4x4_f64 *left, + const struct xrt_matrix_4x4_f64 *right, + struct xrt_matrix_4x4_f64 *result) +{ + Eigen::Matrix4d l = map_matrix_4x4_f64(*left); + Eigen::Matrix4d r = map_matrix_4x4_f64(*right); + + map_matrix_4x4_f64(*result) = l * r; +} + +extern "C" void +m_mat4_f64_orientation(const struct xrt_quat *quat, struct xrt_matrix_4x4_f64 *result) +{ + map_matrix_4x4_f64(*result) = Eigen::Affine3d(copyd(*quat)).matrix(); +} + +extern "C" void +m_mat4_f64_model(const struct xrt_pose *pose, const struct xrt_vec3 *size, struct xrt_matrix_4x4_f64 *result) +{ + Eigen::Vector3d position = copyd(pose->position); + Eigen::Quaterniond orientation = copyd(pose->orientation); + + auto scale = Eigen::Scaling(copyd(size)); + + Eigen::Translation3d translation(position); + Eigen::Affine3d transformation = translation * orientation * scale; + + map_matrix_4x4_f64(*result) = transformation.matrix(); +} + +extern "C" void +m_mat4_f64_view(const struct xrt_pose *pose, struct xrt_matrix_4x4_f64 *result) +{ + Eigen::Vector3d position = copyd(pose->position); + Eigen::Quaterniond orientation = copyd(pose->orientation); + + Eigen::Translation3d translation(position); + Eigen::Affine3d transformation = translation * orientation; + + map_matrix_4x4_f64(*result) = transformation.matrix().inverse(); +} + + /* * * Exported pose functions. @@ -401,6 +543,18 @@ math_pose_invert(const struct xrt_pose *pose, struct xrt_pose *outPose) orientation(*outPose) = newOrientation; } +extern "C" void +math_pose_identity(struct xrt_pose *pose) +{ + pose->position.x = 0.0; + pose->position.y = 0.0; + pose->position.z = 0.0; + pose->orientation.x = 0.0; + pose->orientation.y = 0.0; + pose->orientation.z = 0.0; + pose->orientation.w = 1.0; +} + /*! * Return the result of transforming a point by a pose/transform. */ diff --git a/src/xrt/auxiliary/math/m_documentation.hpp b/src/xrt/auxiliary/math/m_documentation.hpp new file mode 100644 index 000000000..3f839e84e --- /dev/null +++ b/src/xrt/auxiliary/math/m_documentation.hpp @@ -0,0 +1,18 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Header with just documentation. + * @author Ryan Pavlik + * @ingroup aux_math + */ + +#pragma once + + +namespace xrt::auxiliary { +//! C++-only functionality in the Math helper library +namespace math { + +} // namespace math +} // namespace xrt::auxiliary diff --git a/src/xrt/auxiliary/math/m_eigen_interop.hpp b/src/xrt/auxiliary/math/m_eigen_interop.hpp index ed54d5435..bae79df68 100644 --- a/src/xrt/auxiliary/math/m_eigen_interop.hpp +++ b/src/xrt/auxiliary/math/m_eigen_interop.hpp @@ -1,4 +1,4 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -18,6 +18,7 @@ #include #include +namespace xrt::auxiliary::math { /*! * @brief Wrap an internal quaternion struct in an Eigen type, const overload. @@ -70,6 +71,19 @@ map_vec3(struct xrt_vec3 &v) return Eigen::Map{&v.x}; } +/*! + * @brief Wrap an internal 3x3 matrix struct in an Eigen type, non-const + * overload. + * + * Permits zero-overhead manipulation of `xrt_matrix_3x3&` by Eigen routines as + * if it were a `Eigen::Matrix3f&`. + */ +static inline Eigen::Map +map_matrix_3x3(struct xrt_matrix_3x3 &m) +{ + return Eigen::Map(m.v); +} + /*! * @brief Wrap an internal 4x4 matrix struct in an Eigen type, non-const * overload. @@ -83,6 +97,30 @@ map_matrix_4x4(struct xrt_matrix_4x4 &m) return Eigen::Map(m.v); } +/*! + * @brief Wrap an internal 4x4 matrix f64 struct in an Eigen type, const overload. + * + * Permits zero-overhead manipulation of `const xrt_matrix_4x4_f64&` by Eigen routines as if it were a + * `const Eigen::Matrix4d&`. + */ +static inline Eigen::Map +map_matrix_4x4_f64(const struct xrt_matrix_4x4_f64 &m) +{ + return Eigen::Map(m.v); +} + +/*! + * @brief Wrap an internal 4x4 matrix struct in an Eigen type, non-const overload. + * + * Permits zero-overhead manipulation of `xrt_matrix_4x4_f64&` by Eigen routines as if it were a `Eigen::Matrix4d&`. + */ +static inline Eigen::Map +map_matrix_4x4_f64(struct xrt_matrix_4x4_f64 &m) +{ + return Eigen::Map(m.v); +} + + /* * * Pose deconstruction helpers. @@ -124,3 +162,5 @@ position(struct xrt_pose &pose) { return map_vec3(pose.position); } + +} // namespace xrt::auxiliary::math diff --git a/src/xrt/auxiliary/math/m_filter_one_euro.c b/src/xrt/auxiliary/math/m_filter_one_euro.c new file mode 100644 index 000000000..b34182f75 --- /dev/null +++ b/src/xrt/auxiliary/math/m_filter_one_euro.c @@ -0,0 +1,241 @@ +// Copyright 2021, Collabora, Ltd. +// Copyright 2021, Jan Schmidt +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief The "One Euro Filter" for filtering interaction data. + * @author Moses Turner + * @author Jan Schmidt + * @author Ryan Pavlik + * @ingroup aux_math + * + * Based in part on https://github.com/thaytan/OpenHMD/blob/rift-kalman-filter/src/exponential-filter.c + */ + + +#include "m_filter_one_euro.h" + +#include "math/m_mathinclude.h" + +#include "math/m_vec2.h" +#include "math/m_vec3.h" +#include "util/u_time.h" +#include "util/u_misc.h" + +static double +calc_smoothing_alpha(double Fc, double dt) +{ + /* Calculate alpha = (1 / (1 + tau/dt)) where tau = 1.0 / (2 * pi * Fc), + * this is a straight rearrangement with fewer divisions */ + double r = 2.0 * M_PI * Fc * dt; + return r / (r + 1.0); +} + +static double +exp_smooth(double alpha, float y, float prev_y) +{ + return alpha * y + (1.0 - alpha) * prev_y; +} + +static struct xrt_vec2 +exp_smooth_vec2(double alpha, struct xrt_vec2 y, struct xrt_vec2 prev_y) +{ + struct xrt_vec2 scaled_prev = m_vec2_mul_scalar(prev_y, 1.0 - alpha); + struct xrt_vec2 scaled_new = m_vec2_mul_scalar(y, alpha); + return m_vec2_add(scaled_prev, scaled_new); +} + +static struct xrt_vec3 +exp_smooth_vec3(double alpha, struct xrt_vec3 y, struct xrt_vec3 prev_y) +{ + struct xrt_vec3 scaled_prev = m_vec3_mul_scalar(prev_y, 1.0 - alpha); + struct xrt_vec3 scaled_new = m_vec3_mul_scalar(y, alpha); + return m_vec3_add(scaled_prev, scaled_new); +} + +static void +filter_one_euro_init(struct m_filter_one_euro_base *f, double fc_min, double beta, double fc_min_d) +{ + f->fc_min = fc_min; + f->beta = beta; + f->fc_min_d = fc_min_d; + + f->have_prev_y = false; +} + +/// Is this the first sample? If so, please set up the common things. +static bool +filter_one_euro_handle_first_sample(struct m_filter_one_euro_base *f, uint64_t ts, bool commit) +{ + + if (!f->have_prev_y) { + /* First sample - no filtering yet */ + if (commit) { + f->prev_ts = ts; + f->have_prev_y = true; + } + return true; + } + return false; +} + +/** + * @brief Computes and outputs dt, updates the timestamp in the main structure if @p + * commit is true, and computes and returns alpha_d + * + * @param[in,out] f filter base structure + * @param[out] outDt pointer where we should put dt (time since last sample) after computing it. + * @param ts data timestamp + * @param commit true to commit changes to the filter state + * @return alpha_d for filtering derivative + */ +static double +filter_one_euro_compute_alpha_d(struct m_filter_one_euro_base *f, double *outDt, uint64_t ts, bool commit) +{ + double dt = (double)(ts - f->prev_ts) / U_TIME_1S_IN_NS; + if (commit) { + f->prev_ts = ts; + } + *outDt = dt; + return calc_smoothing_alpha(f->fc_min_d, dt); +} + +/** + * @brief Computes and returns alpha + * + * @param[in] f filter base structure + * @param dt Time change in seconds + * @param smoothed_derivative_magnitude the magnitude of the smoothed derivative + * @return alpha for filtering derivative + */ +static double +filter_one_euro_compute_alpha(const struct m_filter_one_euro_base *f, double dt, double smoothed_derivative_magnitude) +{ + double fc_cutoff = f->fc_min + f->beta * smoothed_derivative_magnitude; + return calc_smoothing_alpha(fc_cutoff, dt); +} + + +void +m_filter_euro_f32_init(struct m_filter_euro_f32 *f, double fc_min, double beta, double fc_min_d) +{ + filter_one_euro_init(&f->base, fc_min, beta, fc_min_d); +} + +void +m_filter_f32_run(struct m_filter_euro_f32 *f, uint64_t ts, const float *in_y, float *out_y) +{ + + if (filter_one_euro_handle_first_sample(&f->base, ts, true)) { + /* First sample - no filtering yet */ + f->prev_dy = 0; + f->prev_y = *in_y; + + *out_y = *in_y; + return; + } + + double dt = 0; + double alpha_d = filter_one_euro_compute_alpha_d(&f->base, &dt, ts, true); + + double dy = *in_y - f->prev_y; + + /* Smooth the dy values and use them to calculate the frequency cutoff for the main filter */ + f->prev_dy = exp_smooth(alpha_d, dy, f->prev_dy); + + double dy_mag = fabs(f->prev_dy); + double alpha = filter_one_euro_compute_alpha(&f->base, dt, dy_mag); + + *out_y = f->prev_y = exp_smooth(alpha, *in_y, f->prev_y); +} + +void +m_filter_euro_vec2_init(struct m_filter_euro_vec2 *f, double fc_min, double fc_min_d, double beta) +{ + filter_one_euro_init(&f->base, fc_min, beta, fc_min_d); +} + +void +m_filter_euro_vec2_run(struct m_filter_euro_vec2 *f, uint64_t ts, const struct xrt_vec2 *in_y, struct xrt_vec2 *out_y) +{ + + if (filter_one_euro_handle_first_sample(&f->base, ts, true)) { + /* First sample - no filtering yet */ + U_ZERO(&f->prev_dy); + f->prev_y = *in_y; + *out_y = *in_y; + return; + } + + double dt = 0; + double alpha_d = filter_one_euro_compute_alpha_d(&f->base, &dt, ts, true); + + struct xrt_vec2 dy = m_vec2_sub((*in_y), f->prev_y); + f->prev_dy = exp_smooth_vec2(alpha_d, dy, f->prev_dy); + + double dy_mag = m_vec2_len(f->prev_dy); + double alpha = filter_one_euro_compute_alpha(&f->base, dt, dy_mag); + + /* Smooth the dy values and use them to calculate the frequency cutoff for the main filter */ + f->prev_y = exp_smooth_vec2(alpha, *in_y, f->prev_y); + *out_y = f->prev_y; +} + +void +m_filter_euro_vec2_run_no_commit(struct m_filter_euro_vec2 *f, + uint64_t ts, + const struct xrt_vec2 *in_y, + struct xrt_vec2 *out_y) +{ + + if (filter_one_euro_handle_first_sample(&f->base, ts, false)) { + // First sample - no filtering yet - and we're not committing anything to the filter so just return + *out_y = *in_y; + return; + } + + double dt = 0; + double alpha_d = filter_one_euro_compute_alpha_d(&f->base, &dt, ts, false); + + struct xrt_vec2 dy = m_vec2_sub((*in_y), f->prev_y); + struct xrt_vec2 prev_dy = exp_smooth_vec2(alpha_d, dy, f->prev_dy); + + double dy_mag = m_vec2_len(prev_dy); + + /* Smooth the dy values and use them to calculate the frequency cutoff for the main filter */ + double alpha = filter_one_euro_compute_alpha(&f->base, dt, dy_mag); + *out_y = exp_smooth_vec2(alpha, *in_y, f->prev_y); +} + + +void +m_filter_euro_vec3_init(struct m_filter_euro_vec3 *f, double fc_min, double beta, double fc_min_d) +{ + filter_one_euro_init(&f->base, fc_min, beta, fc_min_d); +} + +void +m_filter_euro_vec3_run(struct m_filter_euro_vec3 *f, uint64_t ts, const struct xrt_vec3 *in_y, struct xrt_vec3 *out_y) +{ + if (filter_one_euro_handle_first_sample(&f->base, ts, true)) { + /* First sample - no filtering yet */ + U_ZERO(&f->prev_dy); + f->prev_y = *in_y; + + *out_y = *in_y; + return; + } + + double dt = 0; + double alpha_d = filter_one_euro_compute_alpha_d(&f->base, &dt, ts, true); + + struct xrt_vec3 dy = m_vec3_sub((*in_y), f->prev_y); + f->prev_dy = exp_smooth_vec3(alpha_d, dy, f->prev_dy); + + double dy_mag = m_vec3_len(f->prev_dy); + double alpha = filter_one_euro_compute_alpha(&f->base, dt, dy_mag); + + /* Smooth the dy values and use them to calculate the frequency cutoff for the main filter */ + f->prev_y = exp_smooth_vec3(alpha, *in_y, f->prev_y); + *out_y = f->prev_y; +} diff --git a/src/xrt/auxiliary/math/m_filter_one_euro.h b/src/xrt/auxiliary/math/m_filter_one_euro.h new file mode 100644 index 000000000..49443ac19 --- /dev/null +++ b/src/xrt/auxiliary/math/m_filter_one_euro.h @@ -0,0 +1,114 @@ +// Copyright 2021, Collabora, Ltd. +// Copyright 2021, Jan Schmidt +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Header for a "One Euro Filter" implementation. + * @author Moses Turner + * @author Jan Schmidt + * @author Ryan Pavlik + * @ingroup aux_math + * + * See the original publication: + * + * Casiez, G., Roussel, N., and Vogel, D. 2012. 1 € filter: a simple speed-based low-pass filter for noisy input in + * interactive systems. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems. Association for + * Computing Machinery, New York, NY, USA, 2527–2530. + * + * Available at: https://hal.inria.fr/hal-00670496/document + */ + +#pragma once + +#include "xrt/xrt_defines.h" +#include "math/m_api.h" + +// Suggestions. These are suitable for head tracking. +#define M_EURO_FILTER_HEAD_TRACKING_FCMIN 30.0 +#define M_EURO_FILTER_HEAD_TRACKING_FCMIN_D 25.0 +#define M_EURO_FILTER_HEAD_TRACKING_BETA 0.6 + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Base data type for One Euro filter instances. + */ +struct m_filter_one_euro_base +{ + /** Minimum frequency cutoff for filter, default = 25.0 */ + float fc_min; + + /** Minimum frequency cutoff for derivative filter, default = 10.0 */ + float fc_min_d; + + /** Beta value for "responsiveness" of filter - default = 0.01 */ + float beta; + + /** true if we have already processed a history sample */ + bool have_prev_y; + + /** Timestamp of previous sample (nanoseconds) */ + uint64_t prev_ts; +}; + +struct m_filter_euro_f32 +{ + /** Base/common data */ + struct m_filter_one_euro_base base; + + /** The previous sample */ + double prev_y; + + /** The previous sample derivative */ + double prev_dy; +}; + +struct m_filter_euro_vec2 +{ + /** Base/common data */ + struct m_filter_one_euro_base base; + + /** The previous sample */ + struct xrt_vec2 prev_y; + + /** The previous sample derivative */ + struct xrt_vec2 prev_dy; +}; + +struct m_filter_euro_vec3 +{ + /** Base/common data */ + struct m_filter_one_euro_base base; + + /** The previous sample */ + struct xrt_vec3 prev_y; + + /** The previous sample derivative */ + struct xrt_vec3 prev_dy; +}; + +void +m_filter_euro_f32_init(struct m_filter_euro_f32 *f, double fc_min, double beta, double fc_min_d); +void +m_filter_euro_f32_run(struct m_filter_euro_f32 *f, uint64_t ts, const float *in_y, float *out_y); + +void +m_filter_euro_vec2_init(struct m_filter_euro_vec2 *f, double fc_min, double beta, double fc_min_d); +void +m_filter_euro_vec2_run(struct m_filter_euro_vec2 *f, uint64_t ts, const struct xrt_vec2 *in_y, struct xrt_vec2 *out_y); +void +m_filter_euro_vec2_run_no_commit(struct m_filter_euro_vec2 *f, + uint64_t ts, + const struct xrt_vec2 *in_y, + struct xrt_vec2 *out_y); + +void +m_filter_euro_vec3_init(struct m_filter_euro_vec3 *f, double fc_min, double beta, double fc_min_d); +void +m_filter_euro_vec3_run(struct m_filter_euro_vec3 *f, uint64_t ts, const struct xrt_vec3 *in_y, struct xrt_vec3 *out_y); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/math/m_imu_3dof.c b/src/xrt/auxiliary/math/m_imu_3dof.c index 9ceceb61c..0a8849a8b 100644 --- a/src/xrt/auxiliary/math/m_imu_3dof.c +++ b/src/xrt/auxiliary/math/m_imu_3dof.c @@ -19,6 +19,7 @@ #include "math/m_mathinclude.h" #include +#include #define DUR_1S_IN_NS (1000 * 1000 * 1000) #define DUR_300MS_IN_NS (300 * 1000 * 1000) @@ -68,6 +69,11 @@ m_imu_3dof_add_vars(struct m_imu_3dof *f, void *root, const char *prefix) u_var_add_ro_vec3_f32(root, &f->grav.error_axis, tmp); snprintf(tmp, sizeof(tmp), "%sgrav.error_angle", prefix); u_var_add_ro_f32(root, &f->grav.error_angle, tmp); + + snprintf(tmp, sizeof(tmp), "%sgyro_bias.value", prefix); + u_var_add_ro_vec3_f32(root, &f->gyro_bias.value, tmp); + snprintf(tmp, sizeof(tmp), "%sgyro_bias.manually_fire", prefix); + u_var_add_bool(root, &f->gyro_bias.manually_fire, tmp); } static void @@ -170,6 +176,26 @@ gravity_correction(struct m_imu_3dof *f, } } +static void +gyro_biasing(struct m_imu_3dof *f, uint64_t timestamp_ns) +{ + if (!f->gyro_bias.manually_fire) { + return; + } + + f->gyro_bias.manually_fire = false; + + uint64_t dur_ns = DUR_300MS_IN_NS; + + struct xrt_vec3 gyro_mean = XRT_VEC3_ZERO; + m_ff_vec3_f32_filter(f->gyro_ff, // Filter + timestamp_ns - dur_ns, // Start time + timestamp_ns, // End time + &gyro_mean); // Results + + f->gyro_bias.value = gyro_mean; +} + void m_imu_3dof_update(struct m_imu_3dof *f, uint64_t timestamp_ns, @@ -183,6 +209,9 @@ m_imu_3dof_update(struct m_imu_3dof *f, return; } + // This code assumes all timestamps makes some forward progress. + assert(timestamp_ns >= f->last.timestamp_ns); + f->last.gyro = *gyro; f->last.accel = *accel; @@ -198,19 +227,20 @@ m_imu_3dof_update(struct m_imu_3dof *f, m_ff_vec3_f32_push(f->word_accel_ff, &world_accel, timestamp_ns); m_ff_vec3_f32_push(f->gyro_ff, gyro, timestamp_ns); - float gyro_length = m_vec3_len(*gyro); + struct xrt_vec3 gyro_biased = m_vec3_sub(*gyro, f->gyro_bias.value); + float gyro_biased_length = m_vec3_len(gyro_biased); - if (gyro_length > 0.0001f) { + if (gyro_biased_length > 0.0001f) { #if 0 math_quat_integrate_velocity(&f->rot, gyro, dt, &f->rot); #else struct xrt_vec3 rot_axis = { - gyro->x / gyro_length, - gyro->y / gyro_length, - gyro->z / gyro_length, + gyro_biased.x / gyro_biased_length, + gyro_biased.y / gyro_biased_length, + gyro_biased.z / gyro_biased_length, }; - float rot_angle = gyro_length * dt; + float rot_angle = gyro_biased_length * dt; struct xrt_quat delta_orient; math_quat_from_angle_vector(rot_angle, &rot_axis, &delta_orient); @@ -220,7 +250,10 @@ m_imu_3dof_update(struct m_imu_3dof *f, } // Gravity correction. - gravity_correction(f, timestamp_ns, accel, gyro, dt, gyro_length); + gravity_correction(f, timestamp_ns, accel, &gyro_biased, dt, gyro_biased_length); + + // Gyro bias calculations. + gyro_biasing(f, timestamp_ns); /* * Mitigate drift due to floating point diff --git a/src/xrt/auxiliary/math/m_imu_3dof.h b/src/xrt/auxiliary/math/m_imu_3dof.h index 0d9e1ef04..074349041 100644 --- a/src/xrt/auxiliary/math/m_imu_3dof.h +++ b/src/xrt/auxiliary/math/m_imu_3dof.h @@ -58,6 +58,13 @@ struct m_imu_3dof struct xrt_vec3 error_axis; float error_angle; } grav; + + // gyro bias correction + struct + { + struct xrt_vec3 value; + bool manually_fire; + } gyro_bias; }; void diff --git a/src/xrt/auxiliary/math/m_mathinclude.h b/src/xrt/auxiliary/math/m_mathinclude.h index 81c3c5a0e..dd51120e4 100644 --- a/src/xrt/auxiliary/math/m_mathinclude.h +++ b/src/xrt/auxiliary/math/m_mathinclude.h @@ -21,6 +21,17 @@ #include #endif + +// Might be missing on Windows. +#ifndef M_PI +#define M_PI (3.14159265358979323846) +#endif + +// Might be missing on Windows. +#ifndef M_PIl +#define M_PIl (3.14159265358979323846264338327950288) +#endif + #ifndef M_1_PI #define M_1_PI (1. / M_PI) #endif diff --git a/src/xrt/auxiliary/math/m_matrix_4x4_f64.h b/src/xrt/auxiliary/math/m_matrix_4x4_f64.h new file mode 100644 index 000000000..a682b645e --- /dev/null +++ b/src/xrt/auxiliary/math/m_matrix_4x4_f64.h @@ -0,0 +1,100 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief C matrix 4x4 f64 math library. + * @author Jakob Bornecrantz + * + * @see xrt_matrix_4x4_f64 + * @ingroup aux_math + */ + +#pragma once + +#include "xrt/xrt_defines.h" + +#include "m_mathinclude.h" + + +#ifdef __cplusplus +extern "C" { +#endif + + +/*! + * Initialize Matrix4x4 F64 with identity. + * + * @relates xrt_matrix_4x4_f64 + * @ingroup aux_math + */ +void +m_mat4_f64_identity(struct xrt_matrix_4x4_f64 *result); + +/*! + * Invert a Matrix4x4 F64. + * + * @relates xrt_matrix_4x4_f64 + * @ingroup aux_math + */ +void +m_mat4_f64_invert(const struct xrt_matrix_4x4_f64 *matrix, struct xrt_matrix_4x4_f64 *result); + +/*! + * Multiply Matrix4x4 F64. + * + * @relates xrt_matrix_4x4_f64 + * @ingroup aux_math + */ +void +m_mat4_f64_multiply(const struct xrt_matrix_4x4_f64 *left, + const struct xrt_matrix_4x4_f64 *right, + struct xrt_matrix_4x4_f64 *result); + +/*! + * Initialize Matrix4x4 F64 with a orientation. + * + * @relates xrt_matrix_4x4_f64 + * @ingroup aux_math + */ +void +m_mat4_f64_orientation(const struct xrt_quat *quat, struct xrt_matrix_4x4_f64 *result); + +/*! + * Initialize Matrix4x4 F64 with a pose and size that can be used as a model matrix. + * + * @relates xrt_matrix_4x4_f64 + * @ingroup aux_math + */ +void +m_mat4_f64_model(const struct xrt_pose *pose, const struct xrt_vec3 *size, struct xrt_matrix_4x4_f64 *result); + +/*! + * Initialize Matrix4x4 F64 with a pose that can be used as a view martix. + * + * @relates xrt_matrix_4x4_f64 + * @ingroup aux_math + */ +void +m_mat4_f64_view(const struct xrt_pose *pose, const struct xrt_vec3 *size, struct xrt_matrix_4x4_f64 *result); + + +#ifdef __cplusplus +} + + +static inline struct xrt_matrix_4x4_f64 // Until clang-format-11 is on the CI. +operator*(const struct xrt_matrix_4x4_f64 &a, const struct xrt_matrix_4x4_f64 &b) +{ + struct xrt_matrix_4x4_f64 ret = {{0}}; + m_mat4_f64_multiply(&l, &r, &ret); + return ret; +} + +static inline void +operator*=(struct xrt_matrix_4x4_f64 &a, const struct xrt_matrix_4x4_f64 &b) +{ + a = a * b; +} + + +#endif diff --git a/src/xrt/auxiliary/math/m_predict.c b/src/xrt/auxiliary/math/m_predict.c index a85928e9e..30149f13c 100644 --- a/src/xrt/auxiliary/math/m_predict.c +++ b/src/xrt/auxiliary/math/m_predict.c @@ -10,6 +10,7 @@ #include "m_api.h" #include "m_vec3.h" #include "m_predict.h" +#include "util/u_trace_marker.h" static void @@ -104,6 +105,7 @@ do_position(const struct xrt_space_relation *rel, void m_predict_relation(const struct xrt_space_relation *rel, double delta_s, struct xrt_space_relation *out_rel) { + XRT_TRACE_MARKER(); enum xrt_space_relation_flags flags = rel->relation_flags; do_orientation(rel, flags, delta_s, out_rel); diff --git a/src/xrt/auxiliary/math/m_predict.h b/src/xrt/auxiliary/math/m_predict.h index 04891e256..dcf31a020 100644 --- a/src/xrt/auxiliary/math/m_predict.h +++ b/src/xrt/auxiliary/math/m_predict.h @@ -12,6 +12,11 @@ #include "xrt/xrt_defines.h" +#ifdef __cplusplus +extern "C" { +#endif + + /*! * Using the given @p xrt_space_relation predicts a new @p xrt_space_relation * @p delta_s into the future. @@ -19,7 +24,12 @@ * Assumes that angular velocity is relative to the space the relation is in, * not relative to relation::pose. * - * @aux_math + * @ingroup aux_math */ void m_predict_relation(const struct xrt_space_relation *rel, double delta_s, struct xrt_space_relation *out_rel); + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/math/m_quatexpmap.cpp b/src/xrt/auxiliary/math/m_quatexpmap.cpp index f5d0dd99e..725bd9a44 100644 --- a/src/xrt/auxiliary/math/m_quatexpmap.cpp +++ b/src/xrt/auxiliary/math/m_quatexpmap.cpp @@ -130,6 +130,8 @@ quat_ln(Eigen::Quaternion const &quat) } // namespace +using namespace xrt::auxiliary::math; + extern "C" void math_quat_integrate_velocity(const struct xrt_quat *quat, const struct xrt_vec3 *ang_vel, diff --git a/src/xrt/auxiliary/math/m_relation_history.cpp b/src/xrt/auxiliary/math/m_relation_history.cpp new file mode 100644 index 000000000..1438c4a80 --- /dev/null +++ b/src/xrt/auxiliary/math/m_relation_history.cpp @@ -0,0 +1,222 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Small utility for keeping track of the history of an xrt_space_relation, ie. for knowing where a HMD or + * controller was in the past. + * @author Moses Turner + * @ingroup drv_ht + */ + +#include +#include +#include +#include +#include +#include +#include "math/m_api.h" +#include "math/m_predict.h" +#include "math/m_vec3.h" +#include "os/os_time.h" +#include "util/u_logging.h" +#include "util/u_trace_marker.h" +#include "xrt/xrt_defines.h" +#include "os/os_threading.h" +#include "util/u_template_historybuf.hpp" + +#include "m_relation_history.h" + + +using namespace xrt::auxiliary::util; + +struct relation_history_entry +{ + struct xrt_space_relation relation; + uint64_t timestamp; +}; + +#define leng 4096 +#define power2 12 +#undef RH_DEBUG + +struct m_relation_history +{ + HistoryBuffer impl; + bool has_first_sample; + struct os_mutex mutex; +}; + + +extern "C" { +void +m_relation_history_create(struct m_relation_history **rh_ptr) +{ + *rh_ptr = U_TYPED_CALLOC(struct m_relation_history); + struct m_relation_history *rh = *rh_ptr; + + rh->has_first_sample = false; + os_mutex_init(&rh->mutex); +#if 0 + struct xrt_space_relation first_relation = {}; + first_relation.pose.orientation.w = 1.0f; // Everything else, including tracked flags, is 0. + m_relation_history_push(rh, &first_relation, os_monotonic_get_ns()); +#endif +} + +void +m_relation_history_push(struct m_relation_history *rh, struct xrt_space_relation *in_relation, uint64_t timestamp) +{ + XRT_TRACE_MARKER(); + struct relation_history_entry rhe; + rhe.relation = *in_relation; + rhe.timestamp = timestamp; + os_mutex_lock(&rh->mutex); + // Don't evaluate the second condition if the length is 0 - rh->impl[0] will be NULL! + if ((!rh->has_first_sample) || (rhe.timestamp > rh->impl[0]->timestamp)) { + // Everything explodes if the timestamps in relation_history aren't monotonically increasing. If we get + // a timestamp that's before the most recent timestamp in the buffer, just don't put it in the history. + rh->impl.push(rhe); + } + rh->has_first_sample = true; + os_mutex_unlock(&rh->mutex); +} + +void +m_relation_history_get(struct m_relation_history *rh, struct xrt_space_relation *out_relation, uint64_t at_timestamp_ns) +{ + XRT_TRACE_MARKER(); + os_mutex_lock(&rh->mutex); + if (rh->has_first_sample == 0) { + // Do nothing. You push nothing to the buffer you get nothing from the buffer. + goto end; + } + + { + uint64_t oldest_in_buffer = rh->impl[rh->impl.length() - 1]->timestamp; + uint64_t newest_in_buffer = rh->impl[0]->timestamp; + + if (at_timestamp_ns > newest_in_buffer) { + // The desired timestamp is after what our buffer contains. + // Aka pose-prediction. + int64_t diff_prediction_ns = 0; + diff_prediction_ns = at_timestamp_ns - newest_in_buffer; + double delta_s = time_ns_to_s(diff_prediction_ns); +#ifdef RH_DEBUG + U_LOG_E("Extrapolating %f s after the head of the buffer!", delta_s); +#endif + m_predict_relation(&rh->impl[0]->relation, delta_s, out_relation); + goto end; + + } else if (at_timestamp_ns < oldest_in_buffer) { + // The desired timestamp is before what our buffer contains. + // Aka a weird edge case where somebody asks for a really old pose and we do our best. + int64_t diff_prediction_ns = 0; + diff_prediction_ns = at_timestamp_ns - oldest_in_buffer; + double delta_s = time_ns_to_s(diff_prediction_ns); +#ifdef RH_DEBUG + U_LOG_E("Extrapolating %f s before the tail of the buffer!", delta_s); +#endif + m_predict_relation(&rh->impl[rh->impl.length() - 1]->relation, delta_s, out_relation); + goto end; + } +#ifdef RH_DEBUG + U_LOG_E("Interpolating within buffer!"); +#endif +#if 0 + // Very slow - O(n) - but easier to read + int idx = 0; + + for (int i = 0; i < rh->impl.length(); i++) { + if (rh->impl[i]->timestamp < at_timestamp_ns) { + // If the entry we're looking at is before the input time + idx = i; + break; + } + } + U_LOG_E("Correct answer is %i", idx); +#else + + // Fast - O(log(n)) - but hard to read + int idx = leng / 2; // 2048 + int step = idx; + + for (int i = power2 - 2; i >= -1; i--) { + uint64_t ts_after = rh->impl[idx - 1]->timestamp; + uint64_t ts_before = rh->impl[idx]->timestamp; + + // This is a little hack because any power of two looks like 0b0001000 (with the 1 in a + // different place for each power). Bit-shift it and it either doubles or halves. In our case it + // halves. step should always be equivalent to pow(2,i). If it's not that's very very bad. + step = step >> 1; +#ifdef RH_DEBUG + assert(step == (int)pow(2, i)); +#endif + + if (idx >= rh->impl.length()) { + // We'd be looking at an uninitialized value. Go back closer to the head of the buffer. + idx -= step; + continue; + } + if ((ts_before < at_timestamp_ns) && (ts_after > at_timestamp_ns)) { + // Found what we're looking for - at_timestamp_ns is between the reading before us and + // the reading after us. Break out of the loop + break; + } + + // This would mean you did the math very wrong. Doesn't happen. + assert(i != -1); + + if (ts_after > at_timestamp_ns) { + // the reading we're looking at is after the reading we want; go closer to the tail of + // the buffer + idx += step; + } else { + // the reading we're looking at is before the reading we want; go closer to the head of + // the buffer + idx -= step; + // Random note: some day, stop using pow(). it's slow, you can do + } + } +#endif + + // Do the thing. + struct xrt_space_relation before = rh->impl[idx]->relation; + struct xrt_space_relation after = rh->impl[idx - 1]->relation; + int64_t diff_before, diff_after = 0; + diff_before = at_timestamp_ns - rh->impl[idx]->timestamp; + diff_after = rh->impl[idx - 1]->timestamp - at_timestamp_ns; + + float amount_to_lerp = (float)diff_before / (float)(diff_before + diff_after); + + // Copy relation flags + out_relation->relation_flags = + (enum xrt_space_relation_flags)(before.relation_flags & after.relation_flags); + + // First-order implementation - just lerp between the before and after + out_relation->pose.position = m_vec3_lerp(before.pose.position, after.pose.position, amount_to_lerp); + math_quat_slerp(&before.pose.orientation, &after.pose.orientation, amount_to_lerp, + &out_relation->pose.orientation); + + //! @todo Does this make any sense? + out_relation->angular_velocity = + m_vec3_lerp(before.angular_velocity, after.angular_velocity, amount_to_lerp); + out_relation->linear_velocity = + m_vec3_lerp(before.linear_velocity, after.linear_velocity, amount_to_lerp); + } +end: + os_mutex_unlock(&rh->mutex); +} + +void +m_relation_history_destroy(struct m_relation_history **rh_ptr) +{ + struct m_relation_history *rh = *rh_ptr; + if (rh == NULL) { + // Do nothing, it's likely already been destroyed + return; + } + os_mutex_destroy(&rh->mutex); + free(rh); + *rh_ptr = NULL; +} +} diff --git a/src/xrt/auxiliary/math/m_relation_history.h b/src/xrt/auxiliary/math/m_relation_history.h new file mode 100644 index 000000000..0ae3be083 --- /dev/null +++ b/src/xrt/auxiliary/math/m_relation_history.h @@ -0,0 +1,54 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Small utility for keeping track of the history of an xrt_space_relation, ie. for knowing where a HMD or + * controller was in the past + * @author Moses Turner + * @ingroup drv_ht + */ + +#include "xrt/xrt_defines.h" +struct m_relation_history; + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * Creates an opaque relation_history object. + * + * @ingroup aux_util + */ +void +m_relation_history_create(struct m_relation_history **rh); + +/*! + * Pushes a new pose to the history - if the history is full, it will also pop a pose out of the other side of the + * buffer. + * + * @ingroup aux_util + */ +void +m_relation_history_push(struct m_relation_history *rh, struct xrt_space_relation *in_relation, uint64_t ts); + +/*! + * Interpolates or extrapolates to the desired timestamp. Read-only operation - doesn't remove anything from the buffer + * or anything like that - you can call this as often as you want. + * + * @ingroup aux_util + */ +void +m_relation_history_get(struct m_relation_history *rh, struct xrt_space_relation *out_relation, uint64_t at_time_ns); + +/*! + * Destroys an opaque relation_history object. + * + * @ingroup aux_util + */ +void +m_relation_history_destroy(struct m_relation_history **rh); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/math/m_space.cpp b/src/xrt/auxiliary/math/m_space.cpp index 73ab3bab1..baecba29b 100644 --- a/src/xrt/auxiliary/math/m_space.cpp +++ b/src/xrt/auxiliary/math/m_space.cpp @@ -126,18 +126,13 @@ make_valid_pose(flags flags, const struct xrt_pose *in_pose, struct xrt_pose *ou if (flags.has_orientation) { out_pose->orientation = in_pose->orientation; } else { - out_pose->orientation.x = 0.0f; - out_pose->orientation.y = 0.0f; - out_pose->orientation.z = 0.0f; - out_pose->orientation.w = 1.0f; + out_pose->orientation = XRT_QUAT_IDENTITY; } if (flags.has_position) { out_pose->position = in_pose->position; } else { - out_pose->position.x = 0.0f; - out_pose->position.y = 0.0f; - out_pose->position.z = 0.0f; + out_pose->position = XRT_VEC3_ZERO; } } @@ -150,9 +145,9 @@ apply_relation(const struct xrt_space_relation *a, flags bf = get_flags(b); flags nf = {}; - struct xrt_pose pose = {}; - struct xrt_vec3 linear_velocity = {}; - struct xrt_vec3 angular_velocity = {}; + struct xrt_pose pose = XRT_POSE_IDENTITY; + struct xrt_vec3 linear_velocity = XRT_VEC3_ZERO; + struct xrt_vec3 angular_velocity = XRT_VEC3_ZERO; /* @@ -161,7 +156,7 @@ apply_relation(const struct xrt_space_relation *a, if (af.has_linear_velocity) { nf.has_linear_velocity = true; - struct xrt_vec3 tmp = {}; + struct xrt_vec3 tmp = XRT_VEC3_ZERO; math_quat_rotate_vec3(&b->pose.orientation, // Base rotation &a->linear_velocity, // In base space &tmp); // Output @@ -180,7 +175,7 @@ apply_relation(const struct xrt_space_relation *a, if (af.has_angular_velocity) { nf.has_angular_velocity = true; - struct xrt_vec3 tmp = {}; + struct xrt_vec3 tmp = XRT_VEC3_ZERO; math_quat_rotate_derivative(&b->pose.orientation, // Base rotation &a->angular_velocity, // In base space @@ -194,10 +189,10 @@ apply_relation(const struct xrt_space_relation *a, nf.has_linear_velocity = true; angular_velocity += b->angular_velocity; - struct xrt_vec3 rotated_position = {}; - struct xrt_vec3 position = {}; - struct xrt_quat orientation = {}; - struct xrt_vec3 tangental_velocity = {}; + struct xrt_vec3 rotated_position = XRT_VEC3_ZERO; + struct xrt_vec3 position = XRT_VEC3_ZERO; + struct xrt_quat orientation = XRT_QUAT_IDENTITY; + struct xrt_vec3 tangental_velocity = XRT_VEC3_ZERO; position = a->pose.position; // In the base space orientation = b->pose.orientation; // Base space @@ -218,8 +213,8 @@ apply_relation(const struct xrt_space_relation *a, * Apply the pose part. */ - struct xrt_pose body_pose = {}; - struct xrt_pose base_pose = {}; + struct xrt_pose body_pose = XRT_POSE_IDENTITY; + struct xrt_pose base_pose = XRT_POSE_IDENTITY; // Only valid poses handled in graph. Flags are determined later. make_valid_pose(af, &a->pose, &body_pose); @@ -284,7 +279,7 @@ void m_space_graph_resolve(const struct xrt_space_graph *xsg, struct xrt_space_relation *out_relation) { if (xsg->num_steps == 0 || has_step_with_no_pose(xsg)) { - U_ZERO(out_relation); + *out_relation = XRT_SPACE_RELATION_ZERO; return; } diff --git a/src/xrt/auxiliary/math/m_space.h b/src/xrt/auxiliary/math/m_space.h index f88f0f2ed..fb569fb4d 100644 --- a/src/xrt/auxiliary/math/m_space.h +++ b/src/xrt/auxiliary/math/m_space.h @@ -21,7 +21,7 @@ extern "C" { /*! - * @ingroup aux_math + * @addtogroup aux_math * @{ */ @@ -65,8 +65,8 @@ m_space_relation_from_pose(const struct xrt_pose *pose, struct xrt_space_relatio struct xrt_space_relation relation = { flags, *pose, - {0, 0, 0}, - {0, 0, 0}, + XRT_VEC3_ZERO, + XRT_VEC3_ZERO, }; *out_relation = relation; @@ -75,10 +75,7 @@ m_space_relation_from_pose(const struct xrt_pose *pose, struct xrt_space_relatio static inline void m_space_relation_ident(struct xrt_space_relation *out_relation) { - struct xrt_pose identity = { - {0, 0, 0, 1}, - {0, 0, 0}, - }; + struct xrt_pose identity = XRT_POSE_IDENTITY; m_space_relation_from_pose(&identity, out_relation); } diff --git a/src/xrt/auxiliary/math/m_vec2.h b/src/xrt/auxiliary/math/m_vec2.h index 5bd54b7b7..32b76e010 100644 --- a/src/xrt/auxiliary/math/m_vec2.h +++ b/src/xrt/auxiliary/math/m_vec2.h @@ -42,6 +42,13 @@ m_vec2_add(struct xrt_vec2 l, struct xrt_vec2 r) return ret; } +static inline struct xrt_vec2 +m_vec2_add_scalar(struct xrt_vec2 l, float r) +{ + struct xrt_vec2 ret = {l.x + r, l.y + r}; + return ret; +} + static inline struct xrt_vec2 m_vec2_sub(struct xrt_vec2 l, struct xrt_vec2 r) { @@ -49,6 +56,13 @@ m_vec2_sub(struct xrt_vec2 l, struct xrt_vec2 r) return ret; } +static inline struct xrt_vec2 +m_vec2_sub_scalar(struct xrt_vec2 l, float r) +{ + struct xrt_vec2 ret = {l.x - r, l.y - r}; + return ret; +} + static inline struct xrt_vec2 m_vec2_div(struct xrt_vec2 l, struct xrt_vec2 r) { @@ -76,12 +90,24 @@ m_vec2_len(struct xrt_vec2 l) return sqrtf(m_vec2_len_sqrd(l)); } +static inline void +m_vec2_normalize(struct xrt_vec2 *inout) +{ + *inout = m_vec2_div_scalar(*inout, m_vec2_len(*inout)); +} + static inline float m_vec2_dot(struct xrt_vec2 l, struct xrt_vec2 r) { return l.x * r.x + l.y * r.y; } +static inline struct xrt_vec2 +m_vec2_lerp(struct xrt_vec2 from, struct xrt_vec2 to, float amount) +{ + // Recommend amount being in [0,1] + return m_vec2_add(m_vec2_mul_scalar(from, 1.0f - amount), m_vec2_mul_scalar(to, amount)); +} #ifdef __cplusplus } diff --git a/src/xrt/auxiliary/math/m_vec3.h b/src/xrt/auxiliary/math/m_vec3.h index a2669c0ed..622077991 100644 --- a/src/xrt/auxiliary/math/m_vec3.h +++ b/src/xrt/auxiliary/math/m_vec3.h @@ -113,6 +113,27 @@ m_vec3_angle(struct xrt_vec3 l, struct xrt_vec3 r) return acosf(dot / lengths); } +static inline struct xrt_vec3 +m_vec3_project(struct xrt_vec3 project_this, struct xrt_vec3 onto_this) +{ + + float amnt = (m_vec3_dot(project_this, onto_this) / m_vec3_len_sqrd(onto_this)); + + return m_vec3_mul_scalar(onto_this, amnt); +} + +static inline struct xrt_vec3 +m_vec3_orthonormalize(struct xrt_vec3 leave_this_alone, struct xrt_vec3 change_this_one) +{ + return m_vec3_normalize(m_vec3_sub(change_this_one, m_vec3_project(change_this_one, leave_this_alone))); +} + +static inline struct xrt_vec3 +m_vec3_lerp(struct xrt_vec3 from, struct xrt_vec3 to, float amount) +{ + // Recommend amount being in [0,1] + return m_vec3_add(m_vec3_mul_scalar(from, 1.0f - amount), m_vec3_mul_scalar(to, amount)); +} #ifdef __cplusplus } diff --git a/src/xrt/auxiliary/meson.build b/src/xrt/auxiliary/meson.build index 2ced9ea8e..2bece1229 100644 --- a/src/xrt/auxiliary/meson.build +++ b/src/xrt/auxiliary/meson.build @@ -11,6 +11,15 @@ u_git_tag_c = vcs_tag( replace_string: '@GIT_DESC@', ) +aux_util_deps = [ xrt_config_have, aux_generated_bindings ] +if libbsd.found() and not get_option('libbsd').disabled() + aux_util_deps += libbsd +endif + +if build_tracing + aux_util_deps += percetto +endif + lib_aux_util = static_library( 'aux_util', files( @@ -31,7 +40,10 @@ lib_aux_util = static_library( 'util/u_format.h', 'util/u_frame.c', 'util/u_frame.h', + 'util/u_generic_callbacks.hpp', 'util/u_git_tag.h', + 'util/u_hand_tracking.c', + 'util/u_hand_tracking.h', 'util/u_handles.c', 'util/u_handles.h', 'util/u_hashmap.cpp', @@ -44,8 +56,6 @@ lib_aux_util = static_library( 'util/u_logging.h', 'util/u_misc.c', 'util/u_misc.h', - 'util/u_render_timing.c', - 'util/u_render_timing.h', 'util/u_sink.h', 'util/u_sink_converter.c', 'util/u_sink_deinterleaver.c', @@ -54,10 +64,20 @@ lib_aux_util = static_library( 'util/u_sink_split.c', 'util/u_time.cpp', 'util/u_time.h', + 'util/u_timing.h', + 'util/u_timing_fake.c', + 'util/u_timing_frame.c', + 'util/u_timing_render.c', + 'util/u_template_historybuf.hpp', + 'util/u_trace_marker.c', + 'util/u_trace_marker.h', 'util/u_var.cpp', 'util/u_var.h', - 'util/u_hand_tracking.c', - 'util/u_hand_tracking.h', + 'util/u_config_json.c', + 'util/u_config_json.h', + 'util/u_verify.h', + 'util/u_process.c', + 'util/u_process.h', ) + [ u_git_tag_c, ], @@ -66,6 +86,7 @@ lib_aux_util = static_library( cjson_include, ], dependencies: [ + aux_util_deps, xrt_config_have, ] ) @@ -144,6 +165,8 @@ lib_aux_math = static_library( 'math/m_eigen_interop.hpp', 'math/m_filter_fifo.c', 'math/m_filter_fifo.h', + 'math/m_filter_one_euro.c', + 'math/m_filter_one_euro.h', 'math/m_hash.cpp', 'math/m_imu_3dof.c', 'math/m_imu_3dof.h', @@ -155,6 +178,8 @@ lib_aux_math = static_library( 'math/m_predict.c', 'math/m_predict.h', 'math/m_quatexpmap.cpp', + 'math/m_relation_history.h', + 'math/m_relation_history.cpp', 'math/m_space.cpp', 'math/m_space.h', 'math/m_vec2.h', @@ -189,6 +214,8 @@ if build_tracking 'tracking/t_debug_hsv_picker.cpp', 'tracking/t_debug_hsv_viewer.cpp', 'tracking/t_file.cpp', + 'tracking/t_frame_cv_mat_wrapper.cpp', + 'tracking/t_frame_cv_mat_wrapper.hpp', 'tracking/t_fusion.hpp', 'tracking/t_helper_debug_sink.hpp', 'tracking/t_hsv_filter.c', @@ -199,6 +226,11 @@ if build_tracking 'tracking/t_tracker_hand.cpp', ] tracking_deps += [opencv] + + if slam.found() + tracking_srcs += ['tracking/t_tracker_slam.cpp'] + tracking_deps += [external_slam] + endif endif lib_aux_tracking = static_library( @@ -217,6 +249,24 @@ aux_tracking = declare_dependency( link_with: lib_aux_tracking, ) +lib_aux_vive = static_library( + 'aux_vive', + files( + 'vive/vive_config.c', + 'vive/vive_config.h', + ), + include_directories: [ + xrt_include, + cjson_include, + ], + dependencies: [zlib, aux_tracking], +) + +aux_vive = declare_dependency( + include_directories: aux_include, + link_with: lib_aux_vive, +) + lib_aux_vk = static_library( 'aux_vk', files( @@ -237,6 +287,6 @@ aux_vk = declare_dependency( ) -all_aux = [aux_util, aux_os, aux_math, aux_tracking, aux_generated_bindings] +all_aux = [aux_util, aux_os, aux_math, aux_tracking, aux_generated_bindings, aux_vive] aux = declare_dependency(dependencies: all_aux) diff --git a/src/xrt/auxiliary/os/os_hid.h b/src/xrt/auxiliary/os/os_hid.h index 89fde6216..f30146879 100644 --- a/src/xrt/auxiliary/os/os_hid.h +++ b/src/xrt/auxiliary/os/os_hid.h @@ -37,6 +37,8 @@ struct os_hid_device int (*set_feature)(struct os_hid_device *hid_dev, const uint8_t *data, size_t size); + int (*get_physical_address)(struct os_hid_device *hid_dev, uint8_t *data, size_t size); + void (*destroy)(struct os_hid_device *hid_dev); }; @@ -104,6 +106,21 @@ os_hid_set_feature(struct os_hid_device *hid_dev, const uint8_t *data, size_t si return hid_dev->set_feature(hid_dev, data, size); } +/*! + * Get the physical address. + * + * For USB devices, the string contains the physical path to the device (the + * USB controller, hubs, ports, etc). For Bluetooth *devices, the string + * contains the hardware (MAC) address of the device. + * + * @public @memberof os_hid_device + */ +static inline int +os_hid_get_physical_address(struct os_hid_device *hid_dev, uint8_t *data, size_t size) +{ + return hid_dev->get_physical_address(hid_dev, data, size); +} + /*! * Close and free the given device. * @@ -119,8 +136,8 @@ os_hid_destroy(struct os_hid_device *hid_dev) /*! * Open the given path as a hidraw device. * - * @public @memberof hid_hidraw - * @relatesalso os_hid_device + * @see hid_hidraw + * @public @memberof os_hid_device */ int os_hid_open_hidraw(const char *path, struct os_hid_device **out_hid); diff --git a/src/xrt/auxiliary/os/os_hid_hidraw.c b/src/xrt/auxiliary/os/os_hid_hidraw.c index ca95b3d2a..56c6e3b3e 100644 --- a/src/xrt/auxiliary/os/os_hid_hidraw.c +++ b/src/xrt/auxiliary/os/os_hid_hidraw.c @@ -89,6 +89,13 @@ os_hidraw_get_feature(struct os_hid_device *ohdev, uint8_t report_num, uint8_t * return ioctl(hrdev->fd, HIDIOCGFEATURE(length), data); } +static int +os_hidraw_get_physical_address(struct os_hid_device *ohdev, uint8_t *data, size_t length) +{ + struct hid_hidraw *hrdev = (struct hid_hidraw *)ohdev; + return ioctl(hrdev->fd, HIDIOCGRAWPHYS(length), data); +} + static int os_hidraw_get_feature_timeout(struct os_hid_device *ohdev, void *data, size_t length, uint32_t timeout) { @@ -138,6 +145,7 @@ os_hid_open_hidraw(const char *path, struct os_hid_device **out_hid) hrdev->base.get_feature = os_hidraw_get_feature; hrdev->base.get_feature_timeout = os_hidraw_get_feature_timeout; hrdev->base.set_feature = os_hidraw_set_feature; + hrdev->base.get_physical_address = os_hidraw_get_physical_address; hrdev->base.destroy = os_hidraw_destroy; hrdev->fd = open(path, O_RDWR); if (hrdev->fd < 0) { diff --git a/src/xrt/auxiliary/os/os_threading.h b/src/xrt/auxiliary/os/os_threading.h index 177bbab8d..bed14157e 100644 --- a/src/xrt/auxiliary/os/os_threading.h +++ b/src/xrt/auxiliary/os/os_threading.h @@ -36,7 +36,7 @@ extern "C" { /*! - * @ingroup aux_os + * @addtogroup aux_os * @{ */ @@ -72,6 +72,12 @@ os_mutex_lock(struct os_mutex *om) pthread_mutex_lock(&om->mutex); } +static inline int +os_mutex_trylock(struct os_mutex *om) +{ + return pthread_mutex_trylock(&om->mutex); +} + /*! * Unlock. */ @@ -98,7 +104,7 @@ os_mutex_destroy(struct os_mutex *om) */ /*! - * A wrapper around a native mutex. + * A wrapper around a native thread. */ struct os_thread { @@ -180,6 +186,33 @@ os_semaphore_release(struct os_semaphore *os) sem_post(&os->sem); } +/*! + * Set @p ts to the current time, plus the timeout_ns value. + * + * Intended for use by the threading code only: the timestamps are not interchangeable with other sources of time. + */ +static inline int +os_semaphore_get_realtime_clock(struct timespec *ts, uint64_t timeout_ns) +{ +#if defined(XRT_OS_WINDOWS) + struct timespec relative; + os_ns_to_timespec(timeout_ns, &relative); + pthread_win32_getabstime_np(ts, &relative); + return 0; +#else + struct timespec now; + if (clock_gettime(CLOCK_REALTIME, &now) < 0) { + assert(false); + return -1; + } + uint64_t now_ns = os_timespec_to_ns(ts); + uint64_t when_ns = timeout_ns + now_ns; + + os_ns_to_timespec(when_ns, ts); + return 0; +#endif +} + /*! * Wait, if @p timeout_ns is zero then waits forever. */ @@ -191,17 +224,11 @@ os_semaphore_wait(struct os_semaphore *os, uint64_t timeout_ns) return; } - struct timespec ts; - if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { + struct timespec abs_timeout; + if (os_semaphore_get_realtime_clock(&abs_timeout, timeout_ns) == -1) { assert(false); } - uint64_t now_ns = os_timespec_to_ns(&ts); - uint64_t when_ns = timeout_ns + now_ns; - - struct timespec abs_timeout = {0, 0}; - os_ns_to_timespec(when_ns, &abs_timeout); - sem_timedwait(&os->sem, &abs_timeout); } diff --git a/src/xrt/auxiliary/os/os_time.h b/src/xrt/auxiliary/os/os_time.h index ff6e818ef..d72077a38 100644 --- a/src/xrt/auxiliary/os/os_time.h +++ b/src/xrt/auxiliary/os/os_time.h @@ -60,13 +60,16 @@ extern "C" { * interoperation with platform APIs. */ - /*! * @brief Sleep the given number of nanoseconds. + * + * Note that on some platforms, this may be somewhat less accurate than you might want. + * On all platforms, the system scheduler has the final say. + * * @ingroup aux_os_time */ static inline void -os_nanosleep(long nsec) +os_nanosleep(int32_t nsec) { #if defined(XRT_OS_LINUX) struct timespec spec; @@ -78,9 +81,82 @@ os_nanosleep(long nsec) #endif } -#ifdef XRT_HAVE_TIMESPEC +/*! + * @brief A structure for storing state as needed for more precise sleeping, mostly for compositor use. + * @ingroup aux_os_time + */ +struct os_precise_sleeper +{ +#if defined(XRT_OS_WINDOWS) + HANDLE timer; +#else + int unused_; +#endif +}; + +/*! + * @brief Initialize members of @ref os_precise_sleeper. + * @public @memberof os_precise_sleeper + */ +static inline void +os_precise_sleeper_init(struct os_precise_sleeper *ops) +{ +#if defined(XRT_OS_WINDOWS) + ops->timer = CreateWaitableTimer(NULL, TRUE, NULL); +#endif +} + +/*! + * @brief De-initialize members of @ref os_precise_sleeper, and free resources, without actually freeing the given + * pointer. + * @public @memberof os_precise_sleeper + */ +static inline void +os_precise_sleeper_deinit(struct os_precise_sleeper *ops) +{ +#if defined(XRT_OS_WINDOWS) + if (ops->timer) { + CloseHandle(ops->timer); + ops->timer = NULL; + } +#endif +} + +/*! + * @brief Sleep the given number of nanoseconds, trying harder to be precise. + * + * On some platforms, there is no way to improve sleep precision easily with some OS-specific state, so we just forward + * to os_nanosleep(). + * + * Note that on all platforms, the system scheduler has the final say. + * + * @public @memberof os_precise_sleeper + */ +static inline void +os_precise_sleeper_nanosleep(struct os_precise_sleeper *ops, int32_t nsec) +{ +#if defined(XRT_OS_WINDOWS) + if (ops->timer) { + LARGE_INTEGER timeperiod; + timeperiod.QuadPart = -(nsec / 100); + if (SetWaitableTimer(ops->timer, &timeperiod, 0, NULL, NULL, FALSE)) { + // OK we could set up the timer, now let's wait. + WaitForSingleObject(ops->timer, INFINITE); + return; + } + } +#endif + // If we fall through from an implementation, or there's no implementation needed for a platform, we just + // delegate to the regular os_nanosleep. + os_nanosleep(nsec); +} + +#if defined(XRT_HAVE_TIMESPEC) /*! * @brief Convert a timespec struct to nanoseconds. + * + * Note that this just does the value combining, no adjustment for epochs is performed. + * * @ingroup aux_os_time_extra */ static inline uint64_t @@ -94,6 +170,8 @@ os_timespec_to_ns(const struct timespec *spec) /*! * @brief Convert an nanosecond integer to a timespec struct. + * + * Note that this just does the value splitting, no adjustment for epochs is performed. * @ingroup aux_os_time_extra */ static inline void @@ -105,7 +183,10 @@ os_ns_to_timespec(uint64_t ns, struct timespec *spec) #endif // XRT_HAVE_TIMESPEC -#ifdef XRT_HAVE_TIMEVAL +#if defined(XRT_HAVE_TIMEVAL) && defined(XRT_OS_LINUX) + +#define OS_NS_PER_USEC (1000) + /*! * @brief Convert a timeval struct to nanoseconds. * @ingroup aux_os_time_extra @@ -115,34 +196,11 @@ os_timeval_to_ns(struct timeval *val) { uint64_t ns = 0; ns += (uint64_t)val->tv_sec * U_1_000_000_000; -#define OS_NS_PER_USEC (1000) ns += (uint64_t)val->tv_usec * OS_NS_PER_USEC; return ns; } #endif // XRT_HAVE_TIMEVAL -#ifdef XRT_OS_WINDOWS -#define CLOCK_MONOTONIC 0 -#define CLOCK_REALTIME 1 - -static int -clock_gettime(int clk_id, struct timespec *spec) -{ - __int64 wintime; - - //! @todo We should be using QueryPerformanceCounter - GetSystemTimeAsFileTime((FILETIME *)&wintime); - // 1jan1601 to 1jan1970 - wintime -= 116444736000000000i64; - // seconds - spec->tv_sec = wintime / 10000000i64; - // nano-seconds - spec->tv_nsec = wintime % 10000000i64 * 100; - - return 0; -} - -#endif // XRT_OS_WINDOWS /*! * @brief Return a monotonic clock in nanoseconds. * @ingroup aux_os_time @@ -150,7 +208,7 @@ clock_gettime(int clk_id, struct timespec *spec) static inline uint64_t os_monotonic_get_ns(void) { -#if defined(XRT_OS_LINUX) || defined(XRT_OS_WINDOWS) +#if defined(XRT_OS_LINUX) struct timespec ts; int ret = clock_gettime(CLOCK_MONOTONIC, &ts); if (ret != 0) { @@ -158,6 +216,19 @@ os_monotonic_get_ns(void) } return os_timespec_to_ns(&ts); +#elif defined(XRT_OS_WINDOWS) + static int64_t ns_per_qpc_tick = 0; + if (ns_per_qpc_tick == 0) { + // Fixed at startup, so we can cache this. + LARGE_INTEGER freq; + QueryPerformanceFrequency(&freq); + ns_per_qpc_tick = U_1_000_000_000 / freq.QuadPart; + } + LARGE_INTEGER qpc; + QueryPerformanceCounter(&qpc); + return qpc.QuadPart * ns_per_qpc_tick; +#else +#error "need port" #endif } diff --git a/src/xrt/auxiliary/tracking/t_calibration.cpp b/src/xrt/auxiliary/tracking/t_calibration.cpp index 8bf79116d..79e80c8f5 100644 --- a/src/xrt/auxiliary/tracking/t_calibration.cpp +++ b/src/xrt/auxiliary/tracking/t_calibration.cpp @@ -14,6 +14,7 @@ #include "util/u_debug.h" #include "util/u_frame.h" #include "util/u_format.h" +#include "util/u_logging.h" #include "tracking/t_tracking.h" #include "tracking/t_calibration_opencv.hpp" @@ -22,12 +23,19 @@ #include #include +#if CV_MAJOR_VERSION >= 4 +#define SB_CHEESBOARD_CORNERS_SUPPORTED +#if CV_MINOR_VERSION >= 3 || CV_MAJOR_VERSION > 4 +#define SB_CHEESBOARD_CORNERS_MARKER_SUPPORTED +#endif +#endif + DEBUG_GET_ONCE_BOOL_OPTION(hsv_filter, "T_DEBUG_HSV_FILTER", false) DEBUG_GET_ONCE_BOOL_OPTION(hsv_picker, "T_DEBUG_HSV_PICKER", false) DEBUG_GET_ONCE_BOOL_OPTION(hsv_viewer, "T_DEBUG_HSV_VIEWER", false) - +namespace xrt::auxiliary::tracking { /* * * Structs @@ -99,6 +107,8 @@ public: cv::Size dims = {8, 6}; enum t_board_pattern pattern = T_BOARD_CHECKERS; float spacing_meters = 0.05; + bool marker; //!< Center board marker for sb_checkers. + bool normalize_image; //!< For SB checkers. } board; struct @@ -325,6 +335,45 @@ do_view_chess(class Calibration &c, struct ViewState &view, cv::Mat &gray, cv::M return found; } +#ifdef SB_CHEESBOARD_CORNERS_SUPPORTED +static bool +do_view_sb_checkers(class Calibration &c, struct ViewState &view, cv::Mat &gray, cv::Mat &rgb) +{ + /* + * Fisheye requires measurement and model to be double, other functions + * requires them to be floats (like cornerSubPix). So we give in + * current_f32 here and convert below. + */ + + int flags = 0; + if (c.board.normalize_image) { + flags += cv::CALIB_CB_NORMALIZE_IMAGE; + } + +#ifdef SB_CHEESBOARD_CORNERS_MARKER_SUPPORTED + if (c.board.marker) { + // Only available in OpenCV 4.3 and above. + flags += cv::CALIB_CB_MARKER; + } +#endif + + bool found = cv::findChessboardCornersSB(gray, // Image + c.board.dims, // patternSize + view.current_f32, // corners + flags); // flags + + // Do the conversion here. + view.current_f64.clear(); // Doesn't effect capacity. + for (const cv::Point2f &p : view.current_f32) { + view.current_f64.emplace_back(double(p.x), double(p.y)); + } + + do_view_coverage(c, view, gray, rgb, found); + + return found; +} +#endif + static bool do_view_circles(class Calibration &c, struct ViewState &view, cv::Mat &gray, cv::Mat &rgb) { @@ -364,6 +413,11 @@ do_view(class Calibration &c, struct ViewState &view, cv::Mat &gray, cv::Mat &rg case T_BOARD_CHECKERS: // found = do_view_chess(c, view, gray, rgb); break; +#ifdef SB_CHEESBOARD_CORNERS_SUPPORTED + case T_BOARD_SB_CHECKERS: // + found = do_view_sb_checkers(c, view, gray, rgb); + break; +#endif case T_BOARD_CIRCLES: // found = do_view_circles(c, view, gray, rgb); break; @@ -405,6 +459,7 @@ build_board_position(class Calibration &c) switch (c.board.pattern) { case T_BOARD_CHECKERS: + case T_BOARD_SB_CHECKERS: case T_BOARD_CIRCLES: // Nothing to do. break; @@ -416,6 +471,7 @@ build_board_position(class Calibration &c) switch (c.board.pattern) { case T_BOARD_CHECKERS: + case T_BOARD_SB_CHECKERS: case T_BOARD_CIRCLES: c.board.model_f32.reserve(rows_num * cols_num); c.board.model_f64.reserve(rows_num * cols_num); @@ -519,7 +575,7 @@ process_stereo_samples(class Calibration &c, int cols, int rows) cv::Size image_size(cols, rows); cv::Size new_image_size(cols, rows); - StereoCameraCalibrationWrapper wrapped = {}; + StereoCameraCalibrationWrapper wrapped = {5}; // We only use five distortion parameters. wrapped.view[0].image_size_pixels.w = image_size.width; wrapped.view[0].image_size_pixels.h = image_size.height; wrapped.view[1].image_size_pixels = wrapped.view[0].image_size_pixels; @@ -1084,6 +1140,21 @@ process_frame_uyvy(class Calibration &c, struct xrt_frame *xf) cv::cvtColor(data_full, c.gray, cv::COLOR_YUV2GRAY_UYVY); } +XRT_NO_INLINE static void +process_frame_rgb(class Calibration &c, struct xrt_frame *xf) +{ + + int w = (int)xf->width; + int h = (int)xf->height; + + cv::Mat rgb_data(h, w, CV_8UC3, xf->data, xf->stride); + ensure_buffers_are_allocated(c, rgb_data.rows, rgb_data.cols); + c.gui.frame->source_sequence = xf->source_sequence; + + cv::cvtColor(rgb_data, c.gray, cv::COLOR_RGB2GRAY); + rgb_data.copyTo(c.gui.rgb); +} + XRT_NO_INLINE static void process_load_image(class Calibration &c, struct xrt_frame *xf) { @@ -1162,6 +1233,7 @@ t_calibration_frame(struct xrt_frame_sink *xsink, struct xrt_frame *xf) case XRT_FORMAT_YUYV422: process_frame_yuyv(c, xf); break; case XRT_FORMAT_UYVY422: process_frame_uyvy(c, xf); break; case XRT_FORMAT_L8: process_frame_l8(c, xf); break; + case XRT_FORMAT_R8G8B8: process_frame_rgb(c, xf); break; default: P("ERROR: Bad format '%s'", u_format_str(xf->format)); make_gui_str(c); @@ -1201,6 +1273,19 @@ t_calibration_stereo_create(struct xrt_frame_context *xfctx, struct xrt_frame_sink *gui, struct xrt_frame_sink **out_sink) { +#ifndef SB_CHEESBOARD_CORNERS_SUPPORTED + if (params->pattern == T_BOARD_SB_CHECKERS) { + U_LOG_E("OpenCV %u.%u doesn't support SB chessboard!", CV_MAJOR_VERSION, CV_MINOR_VERSION); + return -1; + } +#endif +#ifndef SB_CHEESBOARD_CORNERS_MARKER_SUPPORTED + if (params->pattern == T_BOARD_SB_CHECKERS && params->sb_checkers.marker) { + U_LOG_W("OpenCV %u.%u doesn't support SB chessboard marker option!", CV_MAJOR_VERSION, + CV_MINOR_VERSION); + } +#endif + auto &c = *(new Calibration()); // Basic setup. @@ -1222,6 +1307,15 @@ t_calibration_stereo_create(struct xrt_frame_context *xfctx, c.subpixel_enable = params->checkers.subpixel_enable; c.subpixel_size = params->checkers.subpixel_size; break; + case T_BOARD_SB_CHECKERS: + c.board.dims = { + params->sb_checkers.cols, + params->sb_checkers.rows, + }; + c.board.spacing_meters = params->sb_checkers.size_meters; + c.board.marker = params->sb_checkers.marker; + c.board.normalize_image = params->sb_checkers.normalize_image; + break; case T_BOARD_CIRCLES: c.board.dims = { params->circles.cols, @@ -1266,8 +1360,8 @@ t_calibration_stereo_create(struct xrt_frame_context *xfctx, ret = t_debug_hsv_viewer_create(xfctx, *out_sink, out_sink); } - // Ensure we only get yuv, yuyv, uyvy or l8 frames. - u_sink_create_to_yuv_yuyv_uyvy_or_l8(xfctx, *out_sink, out_sink); + // Ensure we only get rgb, yuv, yuyv, uyvy or l8 frames. + u_sink_create_to_rgb_yuv_yuyv_uyvy_or_l8(xfctx, *out_sink, out_sink); // Build the board model. @@ -1291,6 +1385,8 @@ t_calibration_stereo_create(struct xrt_frame_context *xfctx, push_model(c); } #endif + + return ret; } @@ -1416,3 +1512,5 @@ NormalizedCoordsCache::getNormalizedVector(cv::Point2f origCoords) const auto z = -std::sqrt(1.f - pt.dot(pt)); return {pt[0], pt[1], z}; } + +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_calibration_opencv.hpp b/src/xrt/auxiliary/tracking/t_calibration_opencv.hpp index 20bf20009..bea5b1bb6 100644 --- a/src/xrt/auxiliary/tracking/t_calibration_opencv.hpp +++ b/src/xrt/auxiliary/tracking/t_calibration_opencv.hpp @@ -20,6 +20,7 @@ #include #include +namespace xrt::auxiliary::tracking { /*! * @brief Essential calibration data wrapped for C++. @@ -41,7 +42,7 @@ struct CameraCalibrationWrapper : base(calib), image_size_pixels(calib.image_size_pixels), image_size_pixels_cv(calib.image_size_pixels.w, calib.image_size_pixels.h), intrinsics_mat(3, 3, &calib.intrinsics[0][0]), - distortion_mat(XRT_DISTORTION_MAX_DIM, 1, &calib.distortion[0]), + distortion_mat(base.distortion_num, 1, &calib.distortion[0]), distortion_fisheye_mat(4, 1, &calib.distortion_fisheye[0]), use_fisheye(calib.use_fisheye) { assert(isDataStorageValid()); @@ -54,7 +55,7 @@ struct CameraCalibrationWrapper return intrinsics_mat.size() == cv::Size(3, 3) && (double *)intrinsics_mat.data == &(base.intrinsics[0][0]) && - distortion_mat.size() == cv::Size(1, XRT_DISTORTION_MAX_DIM) && + distortion_mat.size() == cv::Size(1, base.distortion_num) && (double *)distortion_mat.data == &(base.distortion[0]) && distortion_fisheye_mat.size() == cv::Size(1, 4) && @@ -80,10 +81,10 @@ struct StereoCameraCalibrationWrapper static t_stereo_camera_calibration * - allocData() + allocData(uint32_t distortion_num) { t_stereo_camera_calibration *data_ptr = NULL; - t_stereo_camera_calibration_alloc(&data_ptr); + t_stereo_camera_calibration_alloc(&data_ptr, distortion_num); return data_ptr; } @@ -101,7 +102,8 @@ struct StereoCameraCalibrationWrapper assert(isDataStorageValid()); } - StereoCameraCalibrationWrapper() : StereoCameraCalibrationWrapper(allocData()) + StereoCameraCalibrationWrapper(uint32_t distortion_num) + : StereoCameraCalibrationWrapper(allocData(distortion_num)) { // The function allocData returns with a ref count of one, @@ -290,3 +292,5 @@ private: cv::Mat_ cacheX_; cv::Mat_ cacheY_; }; + +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_data_utils.c b/src/xrt/auxiliary/tracking/t_data_utils.c index 25b2775cf..1324f7225 100644 --- a/src/xrt/auxiliary/tracking/t_data_utils.c +++ b/src/xrt/auxiliary/tracking/t_data_utils.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -11,12 +11,17 @@ #include "util/u_misc.h" #include +#include void -t_stereo_camera_calibration_alloc(struct t_stereo_camera_calibration **out_c) +t_stereo_camera_calibration_alloc(struct t_stereo_camera_calibration **out_c, uint32_t distortion_num) { + assert(distortion_num == 5 || distortion_num == 14); + struct t_stereo_camera_calibration *c = U_TYPED_CALLOC(struct t_stereo_camera_calibration); + c->view[0].distortion_num = distortion_num; + c->view[1].distortion_num = distortion_num; t_stereo_camera_calibration_reference(out_c, c); } diff --git a/src/xrt/auxiliary/tracking/t_documentation.h b/src/xrt/auxiliary/tracking/t_documentation.h new file mode 100644 index 000000000..f25fb9811 --- /dev/null +++ b/src/xrt/auxiliary/tracking/t_documentation.h @@ -0,0 +1,70 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Documentation-only header. + * @author Pete Black + * @author Jakob Bornecrantz + * @author Ryan Pavlik + * @ingroup aux_tracking + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + + +/*! + * @defgroup aux_tracking Tracking + * @ingroup aux + * @brief Trackers, filters and associated helper code. + * + * + * ### Coordinate system + * + * Right now there is no specific convention on where a tracking systems + * coordinate system is centered, and is something we probably need to figure + * out. Right now the stereo based tracking system used by the PSVR and PSMV + * tracking system is centered on the camera that OpenCV decided is origin. + * + * To go a bit further on the PSVR/PSMV case. Think about a idealized start up + * case, the user is wearing the HMD headset and holding two PSMV controllers. + * The HMD's coordinate system axis are perfectly parallel with the user + * coordinate with the user's coordinate system. Where -Z is forward. The user + * holds the controllers with the ball pointing up and the buttons on the back + * pointing forward. Which if you read the documentation of @ref psmv_device + * will that the axis of the PSMV are also perfectly aligned with the users + * coordinate system. So everything "attached" to the user have it's coordinate + * system parallel to the user's. + * + * The camera on the other hand is looking directly at the user, it's Z-axis and + * X-axis is flipped in relation to the user's. So to compare what is sees to + * what the user sees, everything is rotated 180° around the Y-axis. + */ + +/*! + * @dir auxiliary/tracking + * @ingroup aux + * + * @brief Trackers, filters and associated helper code. + */ + + +#ifdef __cplusplus + +namespace xrt::auxiliary { + /*! + * @brief Namespace used by C++ interfaces in the auxiliary tracking library code. + */ + namespace tracking { + + } // namespace tracking +} // namespace xrt::auxiliary +#endif + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/tracking/t_file.cpp b/src/xrt/auxiliary/tracking/t_file.cpp index 9cb9113ef..848ee42e5 100644 --- a/src/xrt/auxiliary/tracking/t_file.cpp +++ b/src/xrt/auxiliary/tracking/t_file.cpp @@ -13,6 +13,7 @@ #include "util/u_misc.h" #include "util/u_logging.h" + /* * * Pre-declar functions. @@ -31,7 +32,7 @@ write_cv_mat(FILE *f, cv::Mat *m); * Refine and create functions. * */ - +namespace xrt::auxiliary::tracking { RemapPair calibration_get_undistort_map(t_camera_calibration &calib, cv::InputArray rectify_transform_optional, @@ -165,6 +166,7 @@ StereoRectificationMaps::StereoRectificationMaps(t_stereo_camera_calibration *da view[0].rectify = calibration_get_undistort_map(data->view[0], view[0].rotation_mat, view[0].projection_mat); view[1].rectify = calibration_get_undistort_map(data->view[1], view[1].rotation_mat, view[1].projection_mat); } +} // namespace xrt::auxiliary::tracking /* * @@ -175,8 +177,9 @@ StereoRectificationMaps::StereoRectificationMaps(t_stereo_camera_calibration *da extern "C" bool t_stereo_camera_calibration_load_v1(FILE *calib_file, struct t_stereo_camera_calibration **out_data) { + using xrt::auxiliary::tracking::StereoCameraCalibrationWrapper; t_stereo_camera_calibration *data_ptr = NULL; - t_stereo_camera_calibration_alloc(&data_ptr); + t_stereo_camera_calibration_alloc(&data_ptr, 5); // Hardcoded to 5 distortion parameters. StereoCameraCalibrationWrapper wrapped(data_ptr); // Dummy matrix @@ -255,6 +258,7 @@ t_stereo_camera_calibration_load_v1(FILE *calib_file, struct t_stereo_camera_cal extern "C" bool t_stereo_camera_calibration_save_v1(FILE *calib_file, struct t_stereo_camera_calibration *data) { + using xrt::auxiliary::tracking::StereoCameraCalibrationWrapper; StereoCameraCalibrationWrapper wrapped(data); // Dummy matrix cv::Mat dummy; diff --git a/src/xrt/auxiliary/tracking/t_frame_cv_mat_wrapper.cpp b/src/xrt/auxiliary/tracking/t_frame_cv_mat_wrapper.cpp new file mode 100644 index 000000000..122ca51be --- /dev/null +++ b/src/xrt/auxiliary/tracking/t_frame_cv_mat_wrapper.cpp @@ -0,0 +1,116 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Simple @ref xrt_frame wrapper around a @ref cv::Mat. + * @author Jakob Bornecrantz + * @ingroup aux_tracking + */ + +#include "util/u_format.h" + +#include "tracking/t_frame_cv_mat_wrapper.hpp" + + +namespace xrt::auxiliary::tracking { + + +/* + * + * C functions. + * + */ + +extern "C" void +frame_mat_destroy(struct xrt_frame *xf) +{ + FrameMat *fm = (FrameMat *)xf; + delete fm; +} + + +/* + * + * Member functions + * + */ + +void +FrameMat::fillInFields(cv::Mat mat, xrt_format format, const Params ¶ms) +{ + uint32_t width = (uint32_t)mat.cols; + uint32_t height = (uint32_t)mat.rows; + size_t stride = mat.step[0]; + size_t size = stride * height; + + this->matrix = mat; + + // Main wrapping of cv::Mat by frame. + xrt_frame &frame = this->frame; + frame.reference.count = 1; + frame.destroy = frame_mat_destroy; + frame.data = mat.ptr(); + frame.format = format; + frame.width = width; + frame.height = height; + frame.stride = stride; + frame.size = size; + + // Params + frame.timestamp = params.timestamp_ns; + frame.stereo_format = params.stereo_format; +} + +FrameMat::~FrameMat() +{ + // Noop +} + +FrameMat::FrameMat() +{ + // Noop +} + + +/* + * + * Static functions. + * + */ + +void +FrameMat::wrapR8G8B8(cv::Mat mat, xrt_frame **fm_out, const Params /*&&?*/ params) +{ + assert(mat.channels() == 3); + assert(mat.type() == CV_8UC3); + + + FrameMat *fm = new FrameMat(); + fm->fillInFields(mat, XRT_FORMAT_R8G8B8, params); + + // Unreference any old frames. + xrt_frame_reference(fm_out, NULL); + + // Already has a ref count of one. + *fm_out = &fm->frame; +} + +void +FrameMat::wrapL8(cv::Mat mat, xrt_frame **fm_out, const Params /*&&?*/ params) +{ + assert(mat.channels() == 1); + assert(mat.type() == CV_8UC1); + + + FrameMat *fm = new FrameMat(); + fm->fillInFields(mat, XRT_FORMAT_L8, params); + + // Unreference any old frames. + xrt_frame_reference(fm_out, NULL); + + // Already has a ref count of one. + *fm_out = &fm->frame; +} + + +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_frame_cv_mat_wrapper.hpp b/src/xrt/auxiliary/tracking/t_frame_cv_mat_wrapper.hpp new file mode 100644 index 000000000..f459406b1 --- /dev/null +++ b/src/xrt/auxiliary/tracking/t_frame_cv_mat_wrapper.hpp @@ -0,0 +1,74 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Simple @ref xrt_frame wrapper around a @ref cv::Mat. + * @author Jakob Bornecrantz + * @ingroup aux_tracking + */ + +#include "xrt/xrt_frame.h" + +#include + + +namespace xrt::auxiliary::tracking { + + +class FrameMat +{ +public: + /*! + * Additional optional parameters for frame creation. + */ + class Params + { + public: + enum xrt_stereo_format stereo_format; + uint64_t timestamp_ns; + }; + + +public: + // Exposed to the C api. + struct xrt_frame frame = {}; + + // The @ref cv::Mat that holds the data. + cv::Mat matrix = cv::Mat(); + + +public: + /*! + * Only public due to C needed to destroy it. + */ + ~FrameMat(); + + /*! + * Wraps the given @ref cv::Mat assuming it's a 24bit RGB format matrix. + * In all but the most strange cases you probably want the pointer + * pointed to by @ref xf_ptr to be nullptr, if not nullptr it will have + * it's reference decremented so make sure it's a valid pointer. + */ + static void + wrapR8G8B8(cv::Mat mat, xrt_frame **xf_ptr, const Params /*&&?*/ params = {}); + + /*! + * Wraps the given @ref cv::Mat assuming it's a 8bit format matrix. + * In all but the most strange cases you probably want the pointer + * pointed to by @ref xf_ptr to be nullptr, if not nullptr it will have + * it's reference decremented so make sure it's a valid pointer. + */ + static void + wrapL8(cv::Mat mat, xrt_frame **xf_ptr, const Params /*&&?*/ params = {}); + + +private: + FrameMat(); + + void + fillInFields(cv::Mat mat, xrt_format format, const Params ¶ms); +}; + + + +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_fusion.hpp b/src/xrt/auxiliary/tracking/t_fusion.hpp index 7c644e3d6..ffb9c9e0f 100644 --- a/src/xrt/auxiliary/tracking/t_fusion.hpp +++ b/src/xrt/auxiliary/tracking/t_fusion.hpp @@ -22,7 +22,8 @@ #include "flexkalman/PoseState.h" -namespace xrt_fusion { +namespace xrt::auxiliary::tracking { + namespace types = flexkalman::types; using flexkalman::types::Vector; @@ -227,4 +228,5 @@ private: MeasurementVector knownLocationInBodySpace_; MeasurementSquareMatrix covariance_; }; -} // namespace xrt_fusion + +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_helper_debug_sink.hpp b/src/xrt/auxiliary/tracking/t_helper_debug_sink.hpp index 59274351d..275af3b5c 100644 --- a/src/xrt/auxiliary/tracking/t_helper_debug_sink.hpp +++ b/src/xrt/auxiliary/tracking/t_helper_debug_sink.hpp @@ -13,9 +13,11 @@ #error "This header is C++-only." #endif -#include +#include "util/u_sink.h" #include "util/u_frame.h" +#include +namespace xrt::auxiliary::tracking { struct HelperDebugSink { @@ -28,7 +30,7 @@ public: public: Kind kind = AllAvailable; - struct xrt_frame_sink *sink = {}; + struct u_sink_debug usd = {}; struct xrt_frame *frame = {}; cv::Mat rgb[2] = {}; @@ -38,19 +40,21 @@ public: HelperDebugSink(Kind kind) { this->kind = kind; + u_sink_debug_init(&usd); } HelperDebugSink() = delete; ~HelperDebugSink() { + u_sink_debug_destroy(&usd); xrt_frame_reference(&frame, NULL); } void refresh(struct xrt_frame *xf) { - if (sink == NULL) { + if (!u_sink_debug_is_active(&usd)) { return; } @@ -112,15 +116,17 @@ public: void submit() { - if (frame != NULL) { - // Make sure that the cv::Mats doesn't use the data. - rgb[0] = cv::Mat(); - rgb[1] = cv::Mat(); - sink->push_frame(sink, frame); - } + // Make sure that the cv::Mats doesn't use the data. + rgb[0] = cv::Mat(); + rgb[1] = cv::Mat(); + + // Does checking. + u_sink_debug_push_frame(&usd, frame); // We unreference the frame here, downstream is either // done with it or have referenced it themselves. xrt_frame_reference(&frame, NULL); } }; + +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_hsv_filter.c b/src/xrt/auxiliary/tracking/t_hsv_filter.c index 66b44efd9..1a976ea34 100644 --- a/src/xrt/auxiliary/tracking/t_hsv_filter.c +++ b/src/xrt/auxiliary/tracking/t_hsv_filter.c @@ -9,9 +9,11 @@ #include "util/u_var.h" #include "util/u_misc.h" +#include "util/u_sink.h" #include "util/u_debug.h" #include "util/u_frame.h" #include "util/u_format.h" +#include "util/u_trace_marker.h" #include "tracking/t_tracking.h" @@ -134,14 +136,12 @@ struct t_hsv_filter struct xrt_frame_sink *sinks[NUM_CHANNELS]; - struct xrt_frame_sink *debug; struct t_hsv_filter_params params; - struct xrt_frame *frame0; - struct xrt_frame *frame1; - struct xrt_frame *frame2; - struct xrt_frame *frame3; + struct xrt_frame *frames[NUM_CHANNELS]; + + struct u_sink_debug usds[NUM_CHANNELS]; struct t_hsv_filter_optimized_table table; }; @@ -169,12 +169,14 @@ process_sample(struct t_hsv_filter *f, } XRT_NO_INLINE static void -process_frame_yuv(struct t_hsv_filter *f, struct xrt_frame *xf) +hsv_process_frame_yuv(struct t_hsv_filter *f, struct xrt_frame *xf) { - struct xrt_frame *f0 = f->frame0; - struct xrt_frame *f1 = f->frame1; - struct xrt_frame *f2 = f->frame2; - struct xrt_frame *f3 = f->frame3; + SINK_TRACE_MARKER(); + + struct xrt_frame *f0 = f->frames[0]; + struct xrt_frame *f1 = f->frames[1]; + struct xrt_frame *f2 = f->frames[2]; + struct xrt_frame *f3 = f->frames[3]; for (uint32_t y = 0; y < xf->height; y++) { uint8_t *src = (uint8_t *)xf->data + y * xf->stride; @@ -199,12 +201,14 @@ process_frame_yuv(struct t_hsv_filter *f, struct xrt_frame *xf) } XRT_NO_INLINE static void -process_frame_yuyv(struct t_hsv_filter *f, struct xrt_frame *xf) +hsv_process_frame_yuyv(struct t_hsv_filter *f, struct xrt_frame *xf) { - struct xrt_frame *f0 = f->frame0; - struct xrt_frame *f1 = f->frame1; - struct xrt_frame *f2 = f->frame2; - struct xrt_frame *f3 = f->frame3; + SINK_TRACE_MARKER(); + + struct xrt_frame *f0 = f->frames[0]; + struct xrt_frame *f1 = f->frames[1]; + struct xrt_frame *f2 = f->frames[2]; + struct xrt_frame *f3 = f->frames[3]; for (uint32_t y = 0; y < xf->height; y++) { uint8_t *src = (uint8_t *)xf->data + y * xf->stride; @@ -251,74 +255,72 @@ ensure_buf_allocated(struct t_hsv_filter *f, struct xrt_frame *xf) uint32_t w = xf->width; uint32_t h = xf->height; - u_frame_create_one_off(XRT_FORMAT_L8, w, h, &f->frame0); - u_frame_create_one_off(XRT_FORMAT_L8, w, h, &f->frame1); - u_frame_create_one_off(XRT_FORMAT_L8, w, h, &f->frame2); - u_frame_create_one_off(XRT_FORMAT_L8, w, h, &f->frame3); + for (size_t i = 0; i < NUM_CHANNELS; i++) { + u_frame_create_one_off(XRT_FORMAT_L8, w, h, &f->frames[i]); + } } static void -push_buf(struct t_hsv_filter *f, struct xrt_frame *xf, struct xrt_frame_sink *xsink, struct xrt_frame **frame) +push_buf(struct t_hsv_filter *f, + struct xrt_frame *orig_xf, + struct xrt_frame_sink *xsink, + struct u_sink_debug *usd, + struct xrt_frame *xf) { - if (xsink == NULL) { - xrt_frame_reference(frame, NULL); - return; + xf->timestamp = orig_xf->timestamp; + xf->source_id = orig_xf->source_id; + xf->stereo_format = orig_xf->stereo_format; + xf->source_sequence = orig_xf->source_sequence; + xf->source_timestamp = orig_xf->source_timestamp; + + if (xsink != NULL) { + xrt_sink_push_frame(xsink, xf); } - (*frame)->timestamp = xf->timestamp; - (*frame)->source_id = xf->source_id; - (*frame)->stereo_format = xf->stereo_format; - (*frame)->source_sequence = xf->source_sequence; - (*frame)->source_timestamp = xf->source_timestamp; - - xsink->push_frame(xsink, *frame); - - xrt_frame_reference(frame, NULL); + u_sink_debug_push_frame(usd, xf); } static void -push_frame(struct xrt_frame_sink *xsink, struct xrt_frame *xf) +hsv_frame(struct xrt_frame_sink *xsink, struct xrt_frame *xf) { + SINK_TRACE_MARKER(); + struct t_hsv_filter *f = (struct t_hsv_filter *)xsink; switch (xf->format) { case XRT_FORMAT_YUV888: ensure_buf_allocated(f, xf); - process_frame_yuv(f, xf); + hsv_process_frame_yuv(f, xf); break; case XRT_FORMAT_YUYV422: ensure_buf_allocated(f, xf); - process_frame_yuyv(f, xf); + hsv_process_frame_yuyv(f, xf); break; default: U_LOG_E("Bad format '%s'", u_format_str(xf->format)); return; } - push_buf(f, xf, f->sinks[0], &f->frame0); - push_buf(f, xf, f->sinks[1], &f->frame1); - push_buf(f, xf, f->sinks[2], &f->frame2); - push_buf(f, xf, f->sinks[3], &f->frame3); - - // Push the debug frame last, there might be a conversion involved. - if (f->debug != NULL) { - f->debug->push_frame(f->debug, xf); + for (size_t i = 0; i < NUM_CHANNELS; i++) { + push_buf(f, xf, f->sinks[i], &f->usds[i], f->frames[i]); + xrt_frame_reference(&f->frames[i], NULL); } - - assert(f->frame0 == NULL); - assert(f->frame1 == NULL); - assert(f->frame2 == NULL); - assert(f->frame3 == NULL); } static void -break_apart(struct xrt_frame_node *node) -{} +hsv_break_apart(struct xrt_frame_node *node) +{ + // Noop +} static void -destroy(struct xrt_frame_node *node) +hsv_destroy(struct xrt_frame_node *node) { struct t_hsv_filter *f = container_of(node, struct t_hsv_filter, node); u_var_remove_root(f); + for (size_t i = 0; i < ARRAY_SIZE(f->usds); i++) { + u_sink_debug_destroy(&f->usds[i]); + } + free(f); } @@ -329,9 +331,9 @@ t_hsv_filter_create(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_sink) { struct t_hsv_filter *f = U_TYPED_CALLOC(struct t_hsv_filter); - f->base.push_frame = push_frame; - f->node.break_apart = break_apart; - f->node.destroy = destroy; + f->base.push_frame = hsv_frame; + f->node.break_apart = hsv_break_apart; + f->node.destroy = hsv_destroy; f->params = *params; f->sinks[0] = sinks[0]; f->sinks[1] = sinks[1]; @@ -342,12 +344,14 @@ t_hsv_filter_create(struct xrt_frame_context *xfctx, xrt_frame_context_add(xfctx, &f->node); + for (size_t i = 0; i < NUM_CHANNELS; i++) { + u_sink_debug_init(&f->usds[i]); + } u_var_add_root(f, "HSV Filter", true); - u_var_add_sink(f, &f->debug, "Input"); - u_var_add_sink(f, &f->sinks[0], "Red"); - u_var_add_sink(f, &f->sinks[1], "Purple"); - u_var_add_sink(f, &f->sinks[2], "Blue"); - u_var_add_sink(f, &f->sinks[3], "White"); + u_var_add_sink_debug(f, &f->usds[0], "Red"); + u_var_add_sink_debug(f, &f->usds[1], "Purple"); + u_var_add_sink_debug(f, &f->usds[2], "Blue"); + u_var_add_sink_debug(f, &f->usds[3], "White"); *out_sink = &f->base; diff --git a/src/xrt/auxiliary/tracking/t_imu.cpp b/src/xrt/auxiliary/tracking/t_imu.cpp index 9df57331f..ebb586e11 100644 --- a/src/xrt/auxiliary/tracking/t_imu.cpp +++ b/src/xrt/auxiliary/tracking/t_imu.cpp @@ -16,13 +16,15 @@ #include +using xrt::auxiliary::tracking::SimpleIMUFusion; +using namespace xrt::auxiliary::math; struct imu_fusion { public: uint64_t time_ns{0}; - xrt_fusion::SimpleIMUFusion simple_fusion; + SimpleIMUFusion simple_fusion; public: diff --git a/src/xrt/auxiliary/tracking/t_imu_fusion.hpp b/src/xrt/auxiliary/tracking/t_imu_fusion.hpp index d16e3514c..5efab5e93 100644 --- a/src/xrt/auxiliary/tracking/t_imu_fusion.hpp +++ b/src/xrt/auxiliary/tracking/t_imu_fusion.hpp @@ -33,7 +33,11 @@ DEBUG_GET_ONCE_LOG_OPTION(simple_imu_log, "SIMPLE_IMU_LOG", U_LOGGING_WARN) #define SIMPLE_IMU_WARN(...) U_LOG_IFL_W(ll, __VA_ARGS__) #define SIMPLE_IMU_ERROR(...) U_LOG_IFL_E(ll, __VA_ARGS__) -namespace xrt_fusion { +namespace xrt::auxiliary::tracking { + +/*! + * @brief A simple IMU fusion class. + */ class SimpleIMUFusion { public: @@ -304,4 +308,4 @@ SimpleIMUFusion::handleAccel(Eigen::Vector3d const &accel, timepoint_ns timestam return true; } -} // namespace xrt_fusion +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_lowpass.hpp b/src/xrt/auxiliary/tracking/t_lowpass.hpp index 6105ed103..5b8aee262 100644 --- a/src/xrt/auxiliary/tracking/t_lowpass.hpp +++ b/src/xrt/auxiliary/tracking/t_lowpass.hpp @@ -20,8 +20,9 @@ #include -namespace xrt_fusion { -namespace implementation { +namespace xrt::auxiliary::tracking { + +namespace detail { /*! * The shared implementation (between vector and scalar versions) of an * IIR low-pass filter. @@ -99,7 +100,7 @@ namespace implementation { bool initialized{false}; timepoint_ns filter_timestamp_ns{0}; }; -} // namespace implementation +} // namespace detail /*! * A very simple low-pass filter, using a "one-pole infinite impulse response" @@ -172,7 +173,7 @@ public: } private: - implementation::LowPassIIR impl_; + detail::LowPassIIR impl_; }; -} // namespace xrt_fusion +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_lowpass_vector.hpp b/src/xrt/auxiliary/tracking/t_lowpass_vector.hpp index 2e2dbc8fa..c8b6d7f71 100644 --- a/src/xrt/auxiliary/tracking/t_lowpass_vector.hpp +++ b/src/xrt/auxiliary/tracking/t_lowpass_vector.hpp @@ -18,7 +18,7 @@ #include -namespace xrt_fusion { +namespace xrt::auxiliary::tracking { /*! * A very simple low-pass filter, using a "one-pole infinite impulse response" @@ -95,7 +95,7 @@ public: } private: - implementation::LowPassIIR impl_; + detail::LowPassIIR impl_; }; -} // namespace xrt_fusion +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_tracker_hand.cpp b/src/xrt/auxiliary/tracking/t_tracker_hand.cpp index b2482d101..1d4860714 100644 --- a/src/xrt/auxiliary/tracking/t_tracker_hand.cpp +++ b/src/xrt/auxiliary/tracking/t_tracker_hand.cpp @@ -19,11 +19,15 @@ #include "t_tracking.h" #include "tracking/t_calibration_opencv.hpp" +using namespace xrt::auxiliary::tracking; + +//! Namespace for hand tracker implementation +namespace xrt::auxiliary::tracking::hand { /*! * Single camera. * - * @see TrackerPSMV + * @see TrackerHand */ struct View { @@ -236,6 +240,11 @@ run(TrackerHand &t) os_thread_helper_unlock(&t.oth); } + +} // namespace xrt::auxiliary::tracking::hand + +using xrt::auxiliary::tracking::hand::TrackerHand; + extern "C" void * t_ht_run(void *ptr) { @@ -397,7 +406,7 @@ t_hand_create(struct xrt_frame_context *xfctx, u_var_add_root(&t, "Hand Tracker", true); u_var_add_vec3_f32(&t, &t.hand_data[0].hand_relation.pose.position, "hand.tracker.pos.0"); u_var_add_vec3_f32(&t, &t.hand_data[1].hand_relation.pose.position, "hand.tracker.pos.1"); - u_var_add_sink(&t, &t.debug.sink, "Debug"); + u_var_add_sink_debug(&t, &t.debug.usd, "Debug"); *out_sink = &t.sink; *out_xth = &t.base; diff --git a/src/xrt/auxiliary/tracking/t_tracker_psmv.cpp b/src/xrt/auxiliary/tracking/t_tracker_psmv.cpp index 7cd6bf8ed..d18d6fd7e 100644 --- a/src/xrt/auxiliary/tracking/t_tracker_psmv.cpp +++ b/src/xrt/auxiliary/tracking/t_tracker_psmv.cpp @@ -30,6 +30,10 @@ #include #include +using namespace xrt::auxiliary::tracking; + +//! Namespace for PS Move tracking implementation +namespace xrt::auxiliary::tracking::psmv { /*! * Single camera. @@ -106,7 +110,7 @@ struct TrackerPSMV cv::Ptr sbd; - std::unique_ptr filter; + std::unique_ptr filter; xrt_vec3 tracked_object_position; }; @@ -204,14 +208,25 @@ make_lowest_score_finder(FunctionType scoreFunctor) static cv::Point3f world_point_from_blobs(cv::Point2f left, cv::Point2f right, const cv::Matx44d &disparity_to_depth) { - float disp = right.x - left.x; + float disp = left.x - right.x; cv::Vec4d xydw(left.x, left.y, disp, 1.0f); + // Transform cv::Vec4d h_world = disparity_to_depth * xydw; - // Divide by scale to get 3D vector from homogeneous coordinate. we - // also invert x here - cv::Point3f world_point(-h_world[0] / h_world[3], h_world[1] / h_world[3], h_world[2] / h_world[3]); + // Divide by scale to get 3D vector from homogeneous coordinate. + cv::Point3f world_point( // + h_world[0] / h_world[3], // + h_world[1] / h_world[3], // + h_world[2] / h_world[3]); // + + /* + * OpenCV camera space is right handed, -Y up, +Z forwards but + * Monados camera space is right handed, +Y up, -Z forwards so we need + * to invert y and z. + */ + world_point.y = -world_point.y; + world_point.z = -world_point.z; return world_point; } @@ -431,6 +446,9 @@ break_apart(TrackerPSMV &t) os_thread_helper_stop(&t.oth); } +} // namespace xrt::auxiliary::tracking::psmv + +using xrt::auxiliary::tracking::psmv::TrackerPSMV; /* * @@ -534,7 +552,7 @@ t_psmv_create(struct xrt_frame_context *xfctx, t.fusion.rot.y = 0.0f; t.fusion.rot.z = 0.0f; t.fusion.rot.w = 1.0f; - t.filter = xrt_fusion::PSMVFusionInterface::create(); + t.filter = PSMVFusionInterface::create(); ret = os_thread_helper_init(&t.oth); if (ret != 0) { @@ -593,7 +611,7 @@ t_psmv_create(struct xrt_frame_context *xfctx, // Everything is safe, now setup the variable tracking. u_var_add_root(&t, "PSMV Tracker", true); u_var_add_vec3_f32(&t, &t.tracked_object_position, "last.ball.pos"); - u_var_add_sink(&t, &t.debug.sink, "Debug"); + u_var_add_sink_debug(&t, &t.debug.usd, "Debug"); *out_sink = &t.sink; *out_xtmv = &t.base; diff --git a/src/xrt/auxiliary/tracking/t_tracker_psmv_fusion.cpp b/src/xrt/auxiliary/tracking/t_tracker_psmv_fusion.cpp index 5ea065e21..0e45869f5 100644 --- a/src/xrt/auxiliary/tracking/t_tracker_psmv_fusion.cpp +++ b/src/xrt/auxiliary/tracking/t_tracker_psmv_fusion.cpp @@ -28,17 +28,21 @@ #include "flexkalman/PoseState.h" -using State = flexkalman::pose_externalized_rotation::State; -using ProcessModel = flexkalman::PoseSeparatelyDampedConstantVelocityProcessModel; +namespace xrt::auxiliary::tracking { -namespace xrt_fusion { +using namespace xrt::auxiliary::math; -struct TrackingInfo -{ - bool valid{false}; - bool tracked{false}; -}; +//! Anonymous namespace to hide implementation names namespace { + + using State = flexkalman::pose_externalized_rotation::State; + using ProcessModel = flexkalman::PoseSeparatelyDampedConstantVelocityProcessModel; + + struct TrackingInfo + { + bool valid{false}; + bool tracked{false}; + }; class PSMVFusion : public PSMVFusionInterface { public: @@ -70,7 +74,7 @@ namespace { State filter_state; ProcessModel process_model; - xrt_fusion::SimpleIMUFusion imu; + xrt::auxiliary::tracking::SimpleIMUFusion imu; timepoint_ns filter_time_ns{0}; bool tracked{false}; @@ -162,8 +166,8 @@ namespace { if (lever_arm_optional) { lever_arm = map_vec3(*lever_arm_optional).cast(); } - auto measurement = - xrt_fusion::AbsolutePositionLeverArmMeasurement{pos.cast(), lever_arm, variance}; + auto measurement = xrt::auxiliary::tracking::AbsolutePositionLeverArmMeasurement{pos.cast(), + lever_arm, variance}; double resid = measurement.getResidual(filter_state).norm(); if (resid > residual_limit) { @@ -233,4 +237,4 @@ PSMVFusionInterface::create() auto ret = std::make_unique(); return ret; } -} // namespace xrt_fusion +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_tracker_psmv_fusion.hpp b/src/xrt/auxiliary/tracking/t_tracker_psmv_fusion.hpp index 44f907a58..9e4b606ba 100644 --- a/src/xrt/auxiliary/tracking/t_tracker_psmv_fusion.hpp +++ b/src/xrt/auxiliary/tracking/t_tracker_psmv_fusion.hpp @@ -23,7 +23,8 @@ #include -namespace xrt_fusion { +namespace xrt::auxiliary::tracking { + class PSMVFusionInterface { public: @@ -52,4 +53,4 @@ public: virtual void get_prediction(timepoint_ns when_ns, struct xrt_space_relation *out_relation) = 0; }; -} // namespace xrt_fusion +} // namespace xrt::auxiliary::tracking diff --git a/src/xrt/auxiliary/tracking/t_tracker_psvr.cpp b/src/xrt/auxiliary/tracking/t_tracker_psvr.cpp index a598dc888..c4ae4bf3c 100644 --- a/src/xrt/auxiliary/tracking/t_tracker_psvr.cpp +++ b/src/xrt/auxiliary/tracking/t_tracker_psvr.cpp @@ -21,8 +21,10 @@ #include "util/u_var.h" #include "util/u_logging.h" +#include "math/m_mathinclude.h" #include "math/m_api.h" #include "math/m_permutation.h" +#include "math/m_imu_3dof.h" #include "os/os_threading.h" @@ -107,6 +109,11 @@ DEBUG_GET_ONCE_LOG_OPTION(psvr_log, "PSVR_TRACKING_LOG", U_LOGGING_WARN) //#define PSVR_DUMP_FOR_OFFLINE_ANALYSIS //#define PSVR_DUMP_IMU_FOR_OFFLINE_ANALYSIS +using namespace xrt::auxiliary::tracking; + +//! Namespace for PSVR tracking implementation +namespace xrt::auxiliary::tracking::psvr { + typedef enum blob_type { BLOB_TYPE_UNKNOWN, @@ -229,7 +236,7 @@ public: struct { struct xrt_vec3 pos = {}; - struct xrt_quat rot = {}; + struct m_imu_3dof imu_3dof; } fusion; struct @@ -1280,7 +1287,7 @@ sample_line(cv::Mat &src, cv::Point2i start, cv::Point2i end, int *inside_length while (1) { // sample our pixel and see if it is in the interior - if (curr_x > 0 && curr_y > 0) { + if (curr_x > 0 && curr_y > 0 && curr_x < src.cols && curr_y < src.rows) { // cv is row, column uint8_t *val = src.ptr(curr_y, curr_x); @@ -1676,7 +1683,9 @@ process(TrackerPSVR &t, struct xrt_frame *xf) // leds. if (t.merged_points.size() >= PSVR_OPTICAL_SOLVE_THRESH) { Eigen::Quaternionf correction = - rot * Eigen::Quaternionf(t.fusion.rot.w, t.fusion.rot.x, t.fusion.rot.y, t.fusion.rot.z).inverse(); + rot * Eigen::Quaternionf(t.fusion.imu_3dof.rot.w, t.fusion.imu_3dof.rot.x, t.fusion.imu_3dof.rot.y, + t.fusion.imu_3dof.rot.z) + .inverse(); float correction_magnitude = t.target_optical_rotation_correction.angularDistance(correction); @@ -1855,10 +1864,14 @@ get_pose(TrackerPSVR &t, timepoint_ns when_ns, struct xrt_space_relation *out_re //! @todo assuming that orientation is actually //! currently tracked. - out_relation->relation_flags = (enum xrt_space_relation_flags)( - XRT_SPACE_RELATION_POSITION_VALID_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT | - XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT); + out_relation->relation_flags = (enum xrt_space_relation_flags)(XRT_SPACE_RELATION_POSITION_VALID_BIT | + XRT_SPACE_RELATION_POSITION_TRACKED_BIT | + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT); + if (t.done_correction) { + out_relation->relation_flags = (enum xrt_space_relation_flags)( + out_relation->relation_flags | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT); + } os_thread_helper_unlock(&t.oth); } @@ -1873,18 +1886,16 @@ imu_data(TrackerPSVR &t, timepoint_ns timestamp_ns, struct xrt_tracking_sample * return; } if (t.last_imu != 0) { - time_duration_ns delta_ns = timestamp_ns - t.last_imu; - float dt = time_ns_to_s(delta_ns); - // Super simple fusion. - math_quat_integrate_velocity(&t.fusion.rot, &sample->gyro_rad_secs, dt, &t.fusion.rot); + // Update 3DOF fusion + m_imu_3dof_update(&t.fusion.imu_3dof, timestamp_ns, &sample->accel_m_s2, &sample->gyro_rad_secs); } // apply our optical correction to imu rotation // data Eigen::Quaternionf corrected_rot_q = - t.optical_rotation_correction * - Eigen::Quaternionf(t.fusion.rot.w, t.fusion.rot.x, t.fusion.rot.y, t.fusion.rot.z); + t.optical_rotation_correction * Eigen::Quaternionf(t.fusion.imu_3dof.rot.w, t.fusion.imu_3dof.rot.x, + t.fusion.imu_3dof.rot.y, t.fusion.imu_3dof.rot.z); Eigen::Matrix4f corrected_rot = Eigen::Matrix4f::Identity(); corrected_rot.block(0, 0, 3, 3) = corrected_rot_q.toRotationMatrix(); @@ -1939,6 +1950,9 @@ break_apart(TrackerPSVR &t) os_thread_helper_stop(&t.oth); } +} // namespace xrt::auxiliary::tracking::psvr + +using xrt::auxiliary::tracking::psvr::TrackerPSVR; /* * @@ -1989,6 +2003,8 @@ t_psvr_node_destroy(struct xrt_frame_node *node) os_thread_helper_destroy(&t_ptr->oth); + m_imu_3dof_close(&t_ptr->fusion.imu_3dof); + delete t_ptr; } @@ -2034,6 +2050,8 @@ t_psvr_create(struct xrt_frame_context *xfctx, PSVR_INFO("%s", __func__); int ret; + using xrt::auxiliary::tracking::psvr::init_filter; + for (uint32_t i = 0; i < PSVR_NUM_LEDS; i++) { init_filter(t.track_filters[i], PSVR_BLOB_PROCESS_NOISE, PSVR_BLOB_MEASUREMENT_NOISE, 1.0f); } @@ -2095,7 +2113,6 @@ t_psvr_create(struct xrt_frame_context *xfctx, t.sink.push_frame = t_psvr_sink_push_frame; t.node.break_apart = t_psvr_node_break_apart; t.node.destroy = t_psvr_node_destroy; - t.fusion.rot.w = 1.0f; ret = os_thread_helper_init(&t.oth); if (ret != 0) { @@ -2107,17 +2124,19 @@ t_psvr_create(struct xrt_frame_context *xfctx, t.fusion.pos.y = 0.0f; t.fusion.pos.z = 0.0f; - t.fusion.rot.x = 0.0f; - t.fusion.rot.y = 0.0f; - t.fusion.rot.z = 0.0f; - t.fusion.rot.w = 1.0f; + m_imu_3dof_init(&t.fusion.imu_3dof, M_IMU_3DOF_USE_GRAVITY_DUR_20MS); + + t.fusion.imu_3dof.rot.x = 0.0f; + t.fusion.imu_3dof.rot.y = 0.0f; + t.fusion.imu_3dof.rot.z = 0.0f; + t.fusion.imu_3dof.rot.w = 1.0f; xrt_frame_context_add(xfctx, &t.node); // Everything is safe, now setup the variable tracking. u_var_add_root(&t, "PSVR Tracker", true); u_var_add_log_level(&t, &t.ll, "Log level"); - u_var_add_sink(&t, &t.debug.sink, "Debug"); + u_var_add_sink_debug(&t, &t.debug.usd, "Debug"); *out_sink = &t.sink; *out_xtvr = &t.base; diff --git a/src/xrt/auxiliary/tracking/t_tracker_slam.cpp b/src/xrt/auxiliary/tracking/t_tracker_slam.cpp new file mode 100644 index 000000000..a8201b867 --- /dev/null +++ b/src/xrt/auxiliary/tracking/t_tracker_slam.cpp @@ -0,0 +1,346 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief SLAM tracking code. + * @author Mateo de Mayo + * @ingroup aux_tracking + */ + +#include "xrt/xrt_config_have.h" +#include "xrt/xrt_tracking.h" +#include "xrt/xrt_frameserver.h" +#include "util/u_debug.h" +#include "os/os_threading.h" +#include "math/m_space.h" + +#include +#include +#include + +#define SLAM_TRACE(...) U_LOG_IFL_T(t.ll, __VA_ARGS__) +#define SLAM_DEBUG(...) U_LOG_IFL_D(t.ll, __VA_ARGS__) +#define SLAM_INFO(...) U_LOG_IFL_I(t.ll, __VA_ARGS__) +#define SLAM_WARN(...) U_LOG_IFL_W(t.ll, __VA_ARGS__) +#define SLAM_ERROR(...) U_LOG_IFL_E(t.ll, __VA_ARGS__) +#define SLAM_ASSERT(predicate, ...) \ + do { \ + bool p = predicate; \ + if (!p) { \ + U_LOG(U_LOGGING_ERROR, __VA_ARGS__); \ + assert(false && "SLAM_ASSERT failed: " #predicate); \ + exit(EXIT_FAILURE); \ + } \ + } while (false); +#define SLAM_ASSERT_(predicate) SLAM_ASSERT(predicate, "Assertion failed " #predicate) + +// Debug assertions, not vital but useful for finding errors +#ifdef NDEBUG +#define SLAM_DASSERT(predicate, ...) +#define SLAM_DASSERT_(predicate) +#else +#define SLAM_DASSERT(predicate, ...) SLAM_ASSERT(predicate, __VA_ARGS__) +#define SLAM_DASSERT_(predicate) SLAM_ASSERT_(predicate) +#endif + +//! SLAM tracking logging level +DEBUG_GET_ONCE_LOG_OPTION(slam_log, "SLAM_LOG", U_LOGGING_WARN) + +//! Config file path, format is specific to the SLAM implementation in use +DEBUG_GET_ONCE_OPTION(slam_config, "SLAM_CONFIG", NULL) + + +//! Namespace for the interface to the external SLAM tracking system +namespace xrt::auxiliary::tracking::slam { + +using cv::Mat; +using cv::MatAllocator; +using cv::UMatData; +using cv::UMatUsageFlags; + +#define USING_OPENCV_3_3_1 (CV_VERSION_MAJOR == 3 && CV_VERSION_MINOR == 3 && CV_VERSION_REVISION == 1) + +#if defined(XRT_HAVE_KIMERA_SLAM) && !USING_OPENCV_3_3_1 +#warning "Kimera-VIO uses OpenCV 3.3.1, use that to prevent conflicts" +#endif + +//! @todo These defs should make OpenCV 4 work but it wasn't tested against a +//! SLAM system that supports that version yet. +#if CV_VERSION_MAJOR < 4 +#define ACCESS_RW 0 +typedef int AccessFlag; +#define CV_AUTOSTEP 0x7fffffff // From opencv2/core/core_c.h +#else +using cv::ACCESS_RW; +using cv::AccessFlag; +#define CV_AUTOSTEP cv::Mat::AUTO_STEP; +#endif + +/*! + * @brief Wraps a @ref xrt_frame with a `cv::Mat` (conversely to @ref FrameMat). + * + * It works by implementing a `cv::MatAllocator` which determines what to do + * when a `cv::Mat` refcount reaches zero. In that case, it decrements the @ref + * xrt_frame refcount once the `cv::Mat` own refcount has reached zero. + * + * @note a @ref MatFrame `cv::Mat` can wrap a @ref FrameMat @ref xrt_frame, + * which in turns wraps a `cv::Mat`, with little overhead, and that is precisely + * how it is being used in this file when the @ref xrt_frame is a @ref FrameMat. + */ +class MatFrame final : public MatAllocator +{ +public: + //! Wraps a @ref xrt_frame in a `cv::Mat` + Mat + wrap(struct xrt_frame *frame) + { + SLAM_DASSERT_(frame->format == XRT_FORMAT_L8 || frame->format == XRT_FORMAT_R8G8B8); + auto img_type = frame->format == XRT_FORMAT_L8 ? CV_8UC1 : CV_8UC3; + + // Wrap the frame data into a cv::Mat header + cv::Mat img{(int)frame->height, (int)frame->width, img_type, frame->data}; + + // Enable reference counting for a user-allocated cv::Mat (i.e., using existing frame->data) + img.u = this->allocate(img.dims, img.size.p, img.type(), img.data, img.step.p, ACCESS_RW, + cv::USAGE_DEFAULT); + SLAM_DASSERT_(img.u->refcount == 0); + img.addref(); + + // Keep a reference to the xrt_frame in the cv userdata field for when the cv::Mat reference reaches 0 + SLAM_DASSERT_(img.u->userdata == NULL); // Should be default-constructed + xrt_frame_reference((struct xrt_frame **)&img.u->userdata, frame); + + return img; + } + + //! Allocates a `cv::UMatData` object which is in charge of reference counting for a `cv::Mat` + UMatData * + allocate( + int dims, const int *sizes, int type, void *data0, size_t *step, AccessFlag, UMatUsageFlags) const override + { + SLAM_DASSERT_(dims == 2 && sizes && data0 && step && step[0] != CV_AUTOSTEP); + UMatData *u = new UMatData(this); + uchar *data = (uchar *)data0; + u->data = u->origdata = data; + u->size = step[0] * sizes[0]; // Row stride * row count + u->flags |= UMatData::USER_ALLOCATED; // External data + return u; + } + + //! Necessary but unused virtual method for a `cv::MatAllocator` + bool + allocate(UMatData *, AccessFlag, UMatUsageFlags) const override + { + SLAM_ASSERT(false, "Shouldn't be reached"); + return false; + } + + //! When `cv::UMatData` refcount reaches zero this method is called, we just + //! decrement the original @ref xrt_frame refcount as it is the one in charge + //! of the memory. + void + deallocate(UMatData *u) const override + { + SLAM_DASSERT_(u->urefcount == 0 && u->refcount == 0); + SLAM_DASSERT_(u->flags & UMatData::USER_ALLOCATED); + xrt_frame_reference((struct xrt_frame **)&u->userdata, NULL); + delete u; + } +}; + +/*! + * Main implementation of @ref xrt_tracked_slam. This is an adapter class for + * SLAM tracking that wraps an external SLAM implementation. + * + * @implements xrt_tracked_slam + * @implements xrt_frame_node + * @implements xrt_frame_sink + * @implements xrt_imu_sink + */ +struct TrackerSlam +{ + struct xrt_tracked_slam base = {}; + struct xrt_frame_node node = {}; //!< Will be called on destruction + slam_tracker *slam; //!< Pointer to the external SLAM system implementation + + struct xrt_slam_sinks sinks = {}; //!< Pointers to the sinks below + struct xrt_frame_sink left_sink = {}; //!< Sends left camera frames to the SLAM system + struct xrt_frame_sink right_sink = {}; //!< Sends right camera frames to the SLAM system + struct xrt_imu_sink imu_sink = {}; //!< Sends imu samples to the SLAM system + + enum u_logging_level ll; //!< Logging level for the SLAM tracker, set by SLAM_LOG var + struct os_thread_helper oth; //!< Thread where the external SLAM system runs + MatFrame *cv_wrapper; //!< Wraps a xrt_frame in a cv::Mat to send to the SLAM system + + // Used for checking that the timestamps come in order + mutable timepoint_ns last_imu_ts = INT64_MIN; + mutable timepoint_ns last_left_ts = INT64_MIN; + mutable timepoint_ns last_right_ts = INT64_MIN; +}; + +} // namespace xrt::auxiliary::tracking::slam + +using namespace xrt::auxiliary::tracking::slam; + +//! Receive and send IMU samples to the external SLAM system +extern "C" void +t_slam_imu_sink_push(struct xrt_imu_sink *sink, struct xrt_imu_sample *s) +{ + auto &t = *container_of(sink, TrackerSlam, imu_sink); + + timepoint_ns ts = s->timestamp_ns; + xrt_vec3_f64 a = s->accel_m_s2; + xrt_vec3_f64 w = s->gyro_rad_secs; + + //! @todo There are many conversions like these between xrt and + //! slam_tracker.hpp types. Implement a casting mechanism to avoid copies. + imu_sample sample{ts, a.x, a.y, a.z, w.x, w.y, w.z}; + t.slam->push_imu_sample(sample); + SLAM_TRACE("imu t=%ld a=[%f,%f,%f] w=[%f,%f,%f]", ts, a.x, a.y, a.z, w.x, w.y, w.z); + + // Check monotonically increasing timestamps + SLAM_DASSERT(ts > t.last_imu_ts, "Sample (%ld) is older than last (%ld)", ts, t.last_imu_ts) + t.last_imu_ts = ts; +} + +/*! + * @brief Get a space relation tracked by a SLAM system at a specified time. + * + * @todo This function should do pose prediction, currently it is not using @p + * when_ns and just returning the latest tracked pose instead. + */ +extern "C" void +t_slam_get_tracked_pose(struct xrt_tracked_slam *xts, timepoint_ns when_ns, struct xrt_space_relation *out_relation) +{ + auto &t = *container_of(xts, TrackerSlam, base); + pose p{}; + bool dequeued = t.slam->try_dequeue_pose(p); + if (dequeued) { + SLAM_TRACE("pose p=[%f,%f,%f] r=[%f,%f,%f,%f]", p.px, p.py, p.pz, p.rx, p.ry, p.rz, p.rw); + + // Note that any pose correction should happen in the device consuming the tracking + out_relation->pose = {{p.rx, p.ry, p.rz, p.rw}, {p.px, p.py, p.pz}}; + out_relation->relation_flags = (enum xrt_space_relation_flags)( + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_POSITION_VALID_BIT | + XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT); + + } else { + SLAM_TRACE("No poses to dequeue"); + out_relation->relation_flags = XRT_SPACE_RELATION_BITMASK_NONE; + } +} + +//! Push the frame to the external SLAM system +static void +push_frame(const TrackerSlam &t, struct xrt_frame *frame, bool is_left) +{ + // Construct and send the image sample + cv::Mat img = t.cv_wrapper->wrap(frame); + SLAM_DASSERT_(frame->timestamp < INT64_MAX); + img_sample sample{(int64_t)frame->timestamp, img, is_left}; + t.slam->push_frame(sample); + SLAM_TRACE("%s frame t=%lu", is_left ? " left" : "right", frame->timestamp); + + // Check monotonically increasing timestamps + timepoint_ns &last_ts = is_left ? t.last_left_ts : t.last_right_ts; + SLAM_DASSERT(sample.timestamp > last_ts, "Frame (%ld) is older than last (%ld)", sample.timestamp, last_ts); + last_ts = sample.timestamp; +} + +extern "C" void +t_slam_frame_sink_push_left(struct xrt_frame_sink *sink, struct xrt_frame *frame) +{ + auto &t = *container_of(sink, TrackerSlam, left_sink); + push_frame(t, frame, true); +} + +extern "C" void +t_slam_frame_sink_push_right(struct xrt_frame_sink *sink, struct xrt_frame *frame) +{ + auto &t = *container_of(sink, TrackerSlam, right_sink); + push_frame(t, frame, false); +} + +extern "C" void +t_slam_node_break_apart(struct xrt_frame_node *node) +{ + auto &t = *container_of(node, TrackerSlam, node); + t.slam->stop(); + os_thread_helper_stop(&t.oth); + SLAM_DEBUG("SLAM tracker dismantled"); +} + +extern "C" void +t_slam_node_destroy(struct xrt_frame_node *node) +{ + auto t_ptr = container_of(node, TrackerSlam, node); + auto &t = *t_ptr; // Needed by SLAM_DEBUG + SLAM_DEBUG("Destroying SLAM tracker"); + os_thread_helper_destroy(&t_ptr->oth); + delete t_ptr->slam; + delete t_ptr->cv_wrapper; + delete t_ptr; +} + +//! Runs the external SLAM system in a separate thread +extern "C" void * +t_slam_run(void *ptr) +{ + auto &t = *(TrackerSlam *)ptr; + SLAM_DEBUG("SLAM tracker starting"); + t.slam->start(); + return NULL; +} + +//! Starts t_slam_run +extern "C" int +t_slam_start(struct xrt_tracked_slam *xts) +{ + auto &t = *container_of(xts, TrackerSlam, base); + int ret = os_thread_helper_start(&t.oth, t_slam_run, &t); + SLAM_ASSERT(ret == 0, "Unable to start thread"); + SLAM_DEBUG("SLAM tracker started"); + return ret; +} + +extern "C" int +t_slam_create(struct xrt_frame_context *xfctx, struct xrt_tracked_slam **out_xts, struct xrt_slam_sinks **out_sink) +{ + enum u_logging_level ll = debug_get_log_option_slam_log(); + const char *config_file = debug_get_option_slam_config(); + if (!config_file) { + U_LOG_IFL_W(ll, "SLAM tracker requires a config file set with the SLAM_CONFIG environment variable"); + return -1; + } + + auto &t = *(new TrackerSlam{}); + t.ll = ll; + t.cv_wrapper = new MatFrame(); + + t.base.get_tracked_pose = t_slam_get_tracked_pose; + + std::string config_file_string = std::string(config_file); + t.slam = new slam_tracker{config_file_string}; + + t.left_sink.push_frame = t_slam_frame_sink_push_left; + t.right_sink.push_frame = t_slam_frame_sink_push_right; + t.imu_sink.push_imu = t_slam_imu_sink_push; + + t.sinks.left = &t.left_sink; + t.sinks.right = &t.right_sink; + t.sinks.imu = &t.imu_sink; + + t.node.break_apart = t_slam_node_break_apart; + t.node.destroy = t_slam_node_destroy; + + int ret = os_thread_helper_init(&t.oth); + SLAM_ASSERT(ret == 0, "Unable to initialize thread"); + + xrt_frame_context_add(xfctx, &t.node); + + *out_xts = &t.base; + *out_sink = &t.sinks; + + SLAM_DEBUG("SLAM tracker created"); + return 0; +} diff --git a/src/xrt/auxiliary/tracking/t_tracking.h b/src/xrt/auxiliary/tracking/t_tracking.h index 8f621fdec..c8f012999 100644 --- a/src/xrt/auxiliary/tracking/t_tracking.h +++ b/src/xrt/auxiliary/tracking/t_tracking.h @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -21,44 +21,8 @@ extern "C" { #endif - /*! - * @defgroup aux_tracking Tracking - * @ingroup aux - * @brief Trackers, filters and associated helper code. - * - * - * ### Coordinate system - * - * Right now there is no specific convention on where a tracking systems - * coordinate system is centered, and is something we probably need to figure - * out. Right now the stereo based tracking system used by the PSVR and PSMV - * tracking system is centered on the camera that OpenCV decided is origin. - * - * To go a bit further on the PSVR/PSMV case. Think about a idealized start up - * case, the user is wearing the HMD headset and holding two PSMV controllers. - * The HMD's coordinate system axis are perfectly parallel with the user - * coordinate with the user's coordinate system. Where -Z is forward. The user - * holds the controllers with the ball pointing up and the buttons on the back - * pointing forward. Which if you read the documentation of @ref psmv_device - * will that the axis of the PSMV are also perfectly aligned with the users - * coordinate system. So everything "attached" to the user have it's coordinate - * system parallel to the user's. - * - * The camera on the other hand is looking directly at the user, it's Z-axis and - * X-axis is flipped in relation to the user's. So to compare what is sees to - * what the user sees, everything is rotated 180° around the Y-axis. - */ - -/*! - * @dir auxiliary/tracking - * @ingroup aux - * - * @brief Trackers, filters and associated helper code. - */ - -/*! - * @ingroup aux_tracking + * @addtogroup aux_tracking * @{ */ @@ -69,9 +33,11 @@ extern "C" { * */ +struct xrt_slam_sinks; struct xrt_tracked_psmv; struct xrt_tracked_psvr; struct xrt_tracked_hand; +struct xrt_tracked_slam; /* @@ -81,7 +47,7 @@ struct xrt_tracked_hand; */ //! Maximum size of rectilinear distortion coefficient array -#define XRT_DISTORTION_MAX_DIM (5) +#define XRT_DISTORTION_MAX_DIM (14) /*! * @brief Essential calibration data for a single camera, or single lens/sensor @@ -95,8 +61,10 @@ struct t_camera_calibration //! Camera intrinsics matrix double intrinsics[3][3]; - //! Rectilinear distortion coefficients: k1, k2, p1, p2[, k3[, k4, k5, - //! k6[, s1, s2, s3, s4]] + //! Number of distortion parameters (non-fisheye). + size_t distortion_num; + + //! Rectilinear distortion coefficients: k1, k2, p1, p2[, k3[, k4, k5, k6[, s1, s2, s3, s4[, Tx, Ty]]]] double distortion[XRT_DISTORTION_MAX_DIM]; //! Fisheye camera distortion coefficients @@ -131,10 +99,12 @@ struct t_stereo_camera_calibration /*! * Allocates a new stereo calibration data, unreferences the old @p calib. * + * Also initializes view[s]::distortion_num, only 5 and 14 is accepted. + * * @public @memberof t_stereo_camera_calibration */ void -t_stereo_camera_calibration_alloc(struct t_stereo_camera_calibration **calib); +t_stereo_camera_calibration_alloc(struct t_stereo_camera_calibration **calib, uint32_t distortion_num); /*! * Only to be called by @p t_stereo_camera_calibration_reference. @@ -198,7 +168,7 @@ t_stereo_camera_calibration_save_v1(FILE *calib_file, struct t_stereo_camera_cal struct t_convert_table { - uint8_t v[256][256][256][3]; + uint8_t v[256][256][256][3]; // nolint(readability-magic-numbers) }; void @@ -296,7 +266,7 @@ t_hsv_filter_sample(struct t_hsv_filter_optimized_table *t, uint32_t y, uint32_t * Construct an HSV filter sink. * @public @memberof t_hsv_filter * - * @relates xrt_frame_context + * @see xrt_frame_context */ int t_hsv_filter_create(struct xrt_frame_context *xfctx, @@ -357,6 +327,18 @@ t_hand_create(struct xrt_frame_context *xfctx, int t_hand_start(struct xrt_tracked_hand *xth); +/*! + * @public @memberof xrt_tracked_slam + */ +int +t_slam_create(struct xrt_frame_context *xfctx, struct xrt_tracked_slam **out_xts, struct xrt_slam_sinks **out_sink); + +/*! + * @public @memberof xrt_tracked_slam + */ +int +t_slam_start(struct xrt_tracked_slam *xts); + /* * * Camera calibration @@ -369,6 +351,8 @@ t_hand_start(struct xrt_tracked_hand *xth); enum t_board_pattern { T_BOARD_CHECKERS, + //! Sector based checker board, using `cv::findChessboardCornersSB`. + T_BOARD_SB_CHECKERS, T_BOARD_CIRCLES, T_BOARD_ASYMMETRIC_CIRCLES, }; @@ -408,6 +392,16 @@ struct t_calibration_params int subpixel_size; } checkers; + struct + { + int cols; + int rows; + float size_meters; + + bool marker; + bool normalize_image; + } sb_checkers; + struct { int cols; @@ -464,6 +458,13 @@ t_calibration_params_default(struct t_calibration_params *p) p->checkers.subpixel_enable = true; p->checkers.subpixel_size = 5; + // Sector based checker board. + p->sb_checkers.cols = 14; + p->sb_checkers.rows = 9; + p->sb_checkers.size_meters = 0.01206f; + p->sb_checkers.marker = false; + p->sb_checkers.normalize_image = false; + // Symmetrical circles. p->circles.cols = 9; p->circles.rows = 7; @@ -500,7 +501,7 @@ t_calibration_params_default(struct t_calibration_params *p) * @param gui Frame sink * @param out_sink Output: created frame sink. * - * @relates xrt_frame_context + * @see xrt_frame_context */ int t_calibration_stereo_create(struct xrt_frame_context *xfctx, @@ -517,7 +518,7 @@ t_calibration_stereo_create(struct xrt_frame_context *xfctx, */ /*! - * @relates xrt_frame_context + * @see xrt_frame_context */ int t_convert_yuv_or_yuyv_create(struct xrt_frame_sink *next, struct xrt_frame_sink **out_sink); @@ -525,7 +526,7 @@ t_convert_yuv_or_yuyv_create(struct xrt_frame_sink *next, struct xrt_frame_sink /*! - * @relates xrt_frame_context + * @see xrt_frame_context */ int t_debug_hsv_picker_create(struct xrt_frame_context *xfctx, @@ -533,7 +534,7 @@ t_debug_hsv_picker_create(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_sink); /*! - * @relates xrt_frame_context + * @see xrt_frame_context */ int t_debug_hsv_viewer_create(struct xrt_frame_context *xfctx, @@ -541,7 +542,7 @@ t_debug_hsv_viewer_create(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_sink); /*! - * @relates xrt_frame_context + * @see xrt_frame_context */ int t_debug_hsv_filter_create(struct xrt_frame_context *xfctx, diff --git a/src/xrt/auxiliary/util/u_config_json.c b/src/xrt/auxiliary/util/u_config_json.c new file mode 100644 index 000000000..11429c372 --- /dev/null +++ b/src/xrt/auxiliary/util/u_config_json.c @@ -0,0 +1,500 @@ +// Copyright 2019, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Code to manage the settings file. + * @author Jakob Bornecrantz + * @ingroup st_prober + */ + +#include +#include "xrt/xrt_settings.h" +#include "xrt/xrt_config.h" + +#include "util/u_file.h" +#include "util/u_json.h" +#include "util/u_debug.h" + +#include "u_config_json.h" + +#include +#include +#include +#include + +#include "bindings/b_generated_bindings.h" + +DEBUG_GET_ONCE_OPTION(active_config, "P_OVERRIDE_ACTIVE_CONFIG", NULL) + +#define CONFIG_FILE_NAME "config_v0.json" + +void +u_config_json_close(struct u_config_json *json) +{ + if (json->root != NULL) { + cJSON_Delete(json->root); + json->root = NULL; + } + json->file_loaded = false; +} + +void +u_config_json_open_or_create_main_file(struct u_config_json *json) +{ + json->file_loaded = false; +#if defined(XRT_OS_LINUX) && !defined(XRT_OS_ANDROID) + char tmp[1024]; + ssize_t ret = u_file_get_path_in_config_dir(CONFIG_FILE_NAME, tmp, sizeof(tmp)); + if (ret <= 0) { + U_LOG_E( + "Could not load or create config file no $HOME " + "or $XDG_CONFIG_HOME env variables defined"); + return; + } + + FILE *file = u_file_open_file_in_config_dir(CONFIG_FILE_NAME, "r"); + if (file == NULL) { + return; + } + + json->file_loaded = true; + + char *str = u_file_read_content(file); + fclose(file); + if (str == NULL) { + U_LOG_E("Could not read the contents of '%s'!", tmp); + return; + } + + // No config created, ignore. + if (strlen(str) == 0) { + free(str); + return; + } + + json->root = cJSON_Parse(str); + if (json->root == NULL) { + U_LOG_E("Failed to parse JSON in '%s':\n%s\n#######", tmp, str); + U_LOG_E("'%s'", cJSON_GetErrorPtr()); + } + + free(str); +#else + //! @todo implement the underlying u_file_get_path_in_config_dir + return; +#endif +} + +static cJSON * +get_obj(cJSON *json, const char *name) +{ + cJSON *item = cJSON_GetObjectItemCaseSensitive(json, name); + if (item == NULL) { + U_LOG_I("JSON does not contain node '%s'!", name); + } + return item; +} + +XRT_MAYBE_UNUSED static bool +get_obj_bool(cJSON *json, const char *name, bool *out_bool) +{ + if (json == NULL) { + return false; + } + cJSON *item = get_obj(json, name); + if (item == NULL) { + return false; + } + + if (!u_json_get_bool(item, out_bool)) { + U_LOG_E("Failed to parse '%s'!", name); + return false; + } + + return true; +} + +static bool +get_obj_int(cJSON *json, const char *name, int *out_int) +{ + if (json == NULL) { + return false; + } + cJSON *item = get_obj(json, name); + if (item == NULL) { + return false; + } + + if (!u_json_get_int(item, out_int)) { + U_LOG_E("Failed to parse '%s'!", name); + return false; + } + + return true; +} + +static bool +get_obj_float(cJSON *json, const char *name, float *out_float) +{ + if (json == NULL) { + return false; + } + cJSON *item = get_obj(json, name); + if (item == NULL) { + return false; + } + + if (!u_json_get_float(item, out_float)) { + U_LOG_E("Failed to parse '%s'!", name); + return false; + } + + return true; +} + +static bool +get_obj_str(cJSON *json, const char *name, char *array, size_t array_size) +{ + if (json == NULL) { + return false; + } + cJSON *item = get_obj(json, name); + if (item == NULL) { + return false; + } + + if (!u_json_get_string_into_array(item, array, array_size)) { + U_LOG_E("Failed to parse '%s'!", name); + return false; + } + + return true; +} + +static bool +is_json_ok(struct u_config_json *json) +{ + if (json->root == NULL) { + if (json->file_loaded) { + U_LOG_E("Config file was loaded but JSON is not parsed!"); + } else { + U_LOG_I("No config file was loaded!"); + } + return false; + } + + return true; +} + +static void +u_config_json_assign_schema(struct u_config_json *json) +{ + cJSON_DeleteItemFromObject(json->root, "$schema"); + cJSON_AddStringToObject(json->root, "$schema", + "https://monado.pages.freedesktop.org/monado/config_v0.schema.json"); +} + +static bool +parse_active(const char *str, const char *from, enum u_config_json_active_config *out_active) +{ + if (strcmp(str, "none") == 0) { + *out_active = U_ACTIVE_CONFIG_NONE; + } else if (strcmp(str, "tracking") == 0) { + *out_active = U_ACTIVE_CONFIG_TRACKING; + } else if (strcmp(str, "remote") == 0) { + *out_active = U_ACTIVE_CONFIG_REMOTE; + } else { + U_LOG_E("Unknown active config '%s' from %s.", str, from); + *out_active = U_ACTIVE_CONFIG_NONE; + return false; + } + + return true; +} + +void +u_config_json_get_active(struct u_config_json *json, enum u_config_json_active_config *out_active) +{ + const char *str = debug_get_option_active_config(); + if (str != NULL && parse_active(str, "environment", out_active)) { + return; + } + + char tmp[256]; + if (!is_json_ok(json) || !get_obj_str(json->root, "active", tmp, sizeof(tmp))) { + *out_active = U_ACTIVE_CONFIG_NONE; + return; + } + + parse_active(tmp, "json", out_active); +} + +bool +u_config_json_get_remote_port(struct u_config_json *json, int *out_port) +{ + cJSON *t = cJSON_GetObjectItemCaseSensitive(json->root, "remote"); + if (t == NULL) { + U_LOG_E("No remote node"); + return false; + } + + int ver = -1; + if (!get_obj_int(t, "version", &ver)) { + U_LOG_E("Missing version tag!"); + return false; + } + if (ver >= 1) { + U_LOG_E("Unknown version tag '%i'!", ver); + return false; + } + + int port = 0; + if (!get_obj_int(t, "port", &port)) { + return false; + } + + *out_port = port; + + return true; +} + +static cJSON * +open_tracking_settings(struct u_config_json *json) +{ + if (!is_json_ok(json)) { + return NULL; + } + + cJSON *t = cJSON_GetObjectItemCaseSensitive(json->root, "tracking"); + if (t == NULL) { + U_LOG_I("Config file does not contain tracking config"); + return NULL; + } + + int ver = -1; + bool bad = false; + + bad |= !get_obj_int(t, "version", &ver); + if (bad || ver >= 1) { + U_LOG_E("Missing or unknown version tag '%i' in tracking config", ver); + return NULL; + } + + return t; +} + +bool +u_config_json_get_tracking_overrides(struct u_config_json *json, + struct xrt_tracking_override *out_overrides, + size_t *out_num_overrides) +{ + cJSON *t = open_tracking_settings(json); + if (t == NULL) { + return false; + } + + + cJSON *overrides = cJSON_GetObjectItemCaseSensitive(t, "tracking_overrides"); + + *out_num_overrides = 0; + + cJSON *override = NULL; + cJSON_ArrayForEach(override, overrides) + { + bool bad = false; + + struct xrt_tracking_override *o = &out_overrides[(*out_num_overrides)++]; + bad |= !get_obj_str(override, "target_device_serial", o->target_device_serial, XRT_DEVICE_NAME_LEN); + bad |= !get_obj_str(override, "tracker_device_serial", o->tracker_device_serial, XRT_DEVICE_NAME_LEN); + + char override_type[256]; + bad |= !get_obj_str(override, "type", override_type, 256); + if (strncmp(override_type, "direct", 256) == 0) { + o->override_type = XRT_TRACKING_OVERRIDE_DIRECT; + } else if (strncmp(override_type, "attached", 256) == 0) { + o->override_type = XRT_TRACKING_OVERRIDE_ATTACHED; + } + + cJSON *offset = cJSON_GetObjectItemCaseSensitive(override, "offset"); + if (offset) { + cJSON *orientation = cJSON_GetObjectItemCaseSensitive(offset, "orientation"); + bad |= !get_obj_float(orientation, "x", &o->offset.orientation.x); + bad |= !get_obj_float(orientation, "y", &o->offset.orientation.y); + bad |= !get_obj_float(orientation, "z", &o->offset.orientation.z); + bad |= !get_obj_float(orientation, "w", &o->offset.orientation.w); + + cJSON *position = cJSON_GetObjectItemCaseSensitive(offset, "position"); + bad |= !get_obj_float(position, "x", &o->offset.position.x); + bad |= !get_obj_float(position, "y", &o->offset.position.y); + bad |= !get_obj_float(position, "z", &o->offset.position.z); + } else { + o->offset.orientation.w = 1; + } + + char input_name[512] = {'\0'}; + get_obj_str(override, "xrt_input_name", input_name, 512); + o->input_name = xrt_input_name_enum(input_name); + + if (bad) { + *out_num_overrides = 0; + return false; + } + } + return true; +} + +bool +u_config_json_get_tracking_settings(struct u_config_json *json, struct xrt_settings_tracking *s) +{ + cJSON *t = open_tracking_settings(json); + if (t == NULL) { + return false; + } + + char tmp[16]; + + bool bad = false; + + bad |= !get_obj_str(t, "camera_name", s->camera_name, sizeof(s->camera_name)); + bad |= !get_obj_int(t, "camera_mode", &s->camera_mode); + bad |= !get_obj_str(t, "camera_type", tmp, sizeof(tmp)); + bad |= !get_obj_str(t, "calibration_path", s->calibration_path, sizeof(s->calibration_path)); + if (bad) { + return false; + } + + if (strcmp(tmp, "regular_mono") == 0) { + s->camera_type = XRT_SETTINGS_CAMERA_TYPE_REGULAR_MONO; + } else if (strcmp(tmp, "regular_sbs") == 0) { + s->camera_type = XRT_SETTINGS_CAMERA_TYPE_REGULAR_SBS; + } else if (strcmp(tmp, "ps4") == 0) { + s->camera_type = XRT_SETTINGS_CAMERA_TYPE_PS4; + } else if (strcmp(tmp, "leap_motion") == 0) { + s->camera_type = XRT_SETTINGS_CAMERA_TYPE_LEAP_MOTION; + } else { + U_LOG_W("Unknown camera type '%s'", tmp); + return false; + } + + return true; +} + +static void +u_config_json_make_default_root(struct u_config_json *json) +{ + json->root = cJSON_CreateObject(); +} + +static void +u_config_write(struct u_config_json *json) +{ + char *str = cJSON_Print(json->root); + U_LOG_D("%s", str); + + FILE *config_file = u_file_open_file_in_config_dir(CONFIG_FILE_NAME, "w"); + fprintf(config_file, "%s\n", str); + fflush(config_file); + fclose(config_file); + config_file = NULL; + free(str); +} + +void +u_config_json_save_calibration(struct u_config_json *json, struct xrt_settings_tracking *settings) +{ + if (!json->file_loaded) { + u_config_json_make_default_root(json); + } + u_config_json_assign_schema(json); + + cJSON *root = json->root; + + cJSON *t = cJSON_GetObjectItem(root, "tracking"); + if (!t) { + t = cJSON_AddObjectToObject(root, "tracking"); + } + + cJSON_DeleteItemFromObject(t, "version"); + cJSON_AddNumberToObject(t, "version", 0); + + cJSON_DeleteItemFromObject(t, "camera_name"); + cJSON_AddStringToObject(t, "camera_name", settings->camera_name); + + cJSON_DeleteItemFromObject(t, "camera_mode"); + cJSON_AddNumberToObject(t, "camera_mode", settings->camera_mode); + + cJSON_DeleteItemFromObject(t, "camera_type"); + switch (settings->camera_type) { + case XRT_SETTINGS_CAMERA_TYPE_REGULAR_MONO: cJSON_AddStringToObject(t, "camera_type", "regular_mono"); break; + case XRT_SETTINGS_CAMERA_TYPE_REGULAR_SBS: cJSON_AddStringToObject(t, "camera_type", "regular_sbs"); break; + case XRT_SETTINGS_CAMERA_TYPE_PS4: cJSON_AddStringToObject(t, "camera_type", "ps4"); break; + case XRT_SETTINGS_CAMERA_TYPE_LEAP_MOTION: cJSON_AddStringToObject(t, "camera_type", "leap_motion"); break; + } + + cJSON_DeleteItemFromObject(t, "calibration_path"); + cJSON_AddStringToObject(t, "calibration_path", settings->calibration_path); + + u_config_write(json); +} + +static cJSON * +make_pose(struct xrt_pose *pose) +{ + cJSON *json = cJSON_CreateObject(); + + cJSON *o = cJSON_CreateObject(); + cJSON_AddNumberToObject(o, "x", pose->orientation.x); + cJSON_AddNumberToObject(o, "y", pose->orientation.y); + cJSON_AddNumberToObject(o, "z", pose->orientation.z); + cJSON_AddNumberToObject(o, "w", pose->orientation.w); + cJSON_AddItemToObject(json, "orientation", o); + + cJSON *p = cJSON_CreateObject(); + cJSON_AddNumberToObject(p, "x", pose->position.x); + cJSON_AddNumberToObject(p, "y", pose->position.y); + cJSON_AddNumberToObject(p, "z", pose->position.z); + cJSON_AddItemToObject(json, "position", p); + + return json; +} + +void +u_config_json_save_overrides(struct u_config_json *json, struct xrt_tracking_override *overrides, size_t num_overrides) +{ + if (!json->file_loaded) { + u_config_json_make_default_root(json); + } + u_config_json_assign_schema(json); + cJSON *root = json->root; + + cJSON *t = cJSON_GetObjectItem(root, "tracking"); + if (!t) { + t = cJSON_AddObjectToObject(root, "tracking"); + } + + cJSON_DeleteItemFromObject(t, "tracking_overrides"); + cJSON *o = cJSON_AddArrayToObject(t, "tracking_overrides"); + + for (size_t i = 0; i < num_overrides; i++) { + cJSON *entry = cJSON_CreateObject(); + + cJSON_AddStringToObject(entry, "target_device_serial", overrides[i].target_device_serial); + cJSON_AddStringToObject(entry, "tracker_device_serial", overrides[i].tracker_device_serial); + + char override_type[256]; + switch (overrides[i].override_type) { + case XRT_TRACKING_OVERRIDE_DIRECT: strncpy(override_type, "direct", 256); break; + case XRT_TRACKING_OVERRIDE_ATTACHED: strncpy(override_type, "attached", 256); break; + } + cJSON_AddStringToObject(entry, "type", override_type); + + cJSON_AddItemToObject(entry, "offset", make_pose(&overrides[i].offset)); + + const char *input_name_string = xrt_input_name_string(overrides[i].input_name); + cJSON_AddStringToObject(entry, "xrt_input_name", input_name_string); + + cJSON_AddItemToArray(o, entry); + } + + u_config_write(json); +} diff --git a/src/xrt/auxiliary/util/u_config_json.h b/src/xrt/auxiliary/util/u_config_json.h new file mode 100644 index 000000000..6b0740758 --- /dev/null +++ b/src/xrt/auxiliary/util/u_config_json.h @@ -0,0 +1,105 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Code to manage the settings file. + * @author Christoph Haag + * @ingroup aux_util + */ + +#pragma once + +#include "util/u_json.h" +#include "xrt/xrt_settings.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * What config is currently active in the config file. + */ +enum u_config_json_active_config +{ + U_ACTIVE_CONFIG_NONE = 0, + U_ACTIVE_CONFIG_TRACKING = 1, + U_ACTIVE_CONFIG_REMOTE = 2, +}; + +struct u_config_json +{ + //! For error reporting, was it loaded but not parsed? + bool file_loaded; + + cJSON *root; +}; + +void +u_config_json_close(struct u_config_json *json); + +/*! + * Load the JSON config file. + * + * @ingroup aux_util + */ +void +u_config_json_open_or_create_main_file(struct u_config_json *json); + +/*! + * Writes back calibration settings to the main config file. + * + * @ingroup aux_util + */ +void +u_config_json_save_calibration(struct u_config_json *json, struct xrt_settings_tracking *settings); + +/*! + * Writes back tracking override settings to the main config file. + * + * @ingroup aux_util + */ +void +u_config_json_save_overrides(struct u_config_json *json, struct xrt_tracking_override *overrides, size_t num_overrides); + +/*! + * Read from the JSON loaded json config file and returns the active config, + * can be overridden by `P_OVERRIDE_ACTIVE_CONFIG` envirmental variable. + * + * @ingroup aux_util + */ +void +u_config_json_get_active(struct u_config_json *json, enum u_config_json_active_config *out_active); + +/*! + * Extract tracking settings from the JSON. + * + * @ingroup aux_util + * @relatesalso xrt_settings_tracking + */ +bool +u_config_json_get_tracking_settings(struct u_config_json *json, struct xrt_settings_tracking *s); + +/*! + * Extract tracking override settings from the JSON. + * + * Caller allocates an array of XRT_MAX_TRACKING_OVERRIDES tracking_override. + * + * @ingroup aux_util + * @relatesalso xrt_settings_tracking + */ +bool +u_config_json_get_tracking_overrides(struct u_config_json *json, + struct xrt_tracking_override *out_overrides, + size_t *out_num_overrides); + +/*! + * Extract remote settings from the JSON. + * + * @ingroup aux_util + */ +bool +u_config_json_get_remote_port(struct u_config_json *json, int *out_port); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/util/u_device.c b/src/xrt/auxiliary/util/u_device.c index 27d131c2c..f0854bd8c 100644 --- a/src/xrt/auxiliary/util/u_device.c +++ b/src/xrt/auxiliary/util/u_device.c @@ -1,9 +1,11 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Misc helpers for device drivers. * @author Jakob Bornecrantz + * @author Ryan Pavlik + * @author Moses Turner * @ingroup aux_util */ @@ -130,6 +132,35 @@ u_device_dump_config(struct xrt_device *xdev, const char *prefix, const char *pr * */ +bool +u_extents_2d_split_side_by_side(struct xrt_device *xdev, const struct u_extents_2d *extents) +{ + uint32_t eye_w_pixels = extents->w_pixels / 2; + uint32_t eye_h_pixels = extents->h_pixels; + + xdev->hmd->screens[0].w_pixels = extents->w_pixels; + xdev->hmd->screens[0].h_pixels = extents->h_pixels; + + // Left + xdev->hmd->views[0].display.w_pixels = eye_w_pixels; + xdev->hmd->views[0].display.h_pixels = eye_h_pixels; + xdev->hmd->views[0].viewport.x_pixels = 0; + xdev->hmd->views[0].viewport.y_pixels = 0; + xdev->hmd->views[0].viewport.w_pixels = eye_w_pixels; + xdev->hmd->views[0].viewport.h_pixels = eye_h_pixels; + xdev->hmd->views[0].rot = u_device_rotation_ident; + + // Right + xdev->hmd->views[1].display.w_pixels = eye_w_pixels; + xdev->hmd->views[1].display.h_pixels = eye_h_pixels; + xdev->hmd->views[1].viewport.x_pixels = eye_w_pixels; + xdev->hmd->views[1].viewport.y_pixels = 0; + xdev->hmd->views[1].viewport.w_pixels = eye_w_pixels; + xdev->hmd->views[1].viewport.h_pixels = eye_h_pixels; + xdev->hmd->views[1].rot = u_device_rotation_ident; + return true; +} + bool u_device_setup_split_side_by_side(struct xrt_device *xdev, const struct u_device_simple_info *info) { @@ -149,7 +180,10 @@ u_device_setup_split_side_by_side(struct xrt_device *xdev, const struct u_device }; // Common - xdev->hmd->blend_mode = XRT_BLEND_MODE_OPAQUE; + size_t idx = 0; + xdev->hmd->blend_modes[idx++] = XRT_BLEND_MODE_OPAQUE; + xdev->hmd->num_blend_modes = idx; + if (xdev->hmd->distortion.models == 0) { xdev->hmd->distortion.models = XRT_DISTORTION_MODEL_NONE; xdev->hmd->distortion.preferred = XRT_DISTORTION_MODEL_NONE; @@ -392,3 +426,27 @@ u_device_setup_tracking_origins(struct xrt_device *head, apply_offset(&right->tracking_origin->offset.position, global_tracking_origin_offset); } } + +void +u_device_get_view_pose(const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) +{ + struct xrt_pose pose = XRT_POSE_IDENTITY; + bool adjust = view_index == 0; + + pose.position.x = eye_relation->x / 2.0f; + pose.position.y = eye_relation->y / 2.0f; + pose.position.z = eye_relation->z / 2.0f; + + // Adjust for left/right while also making sure there aren't any -0.f. + if (pose.position.x > 0.0f && adjust) { + pose.position.x = -pose.position.x; + } + if (pose.position.y > 0.0f && adjust) { + pose.position.y = -pose.position.y; + } + if (pose.position.z > 0.0f && adjust) { + pose.position.z = -pose.position.z; + } + + *out_pose = pose; +} diff --git a/src/xrt/auxiliary/util/u_device.h b/src/xrt/auxiliary/util/u_device.h index c3251dfc5..4587dfaee 100644 --- a/src/xrt/auxiliary/util/u_device.h +++ b/src/xrt/auxiliary/util/u_device.h @@ -1,9 +1,11 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Misc helpers for device drivers. * @author Jakob Bornecrantz + * @author Ryan Pavlik + * @author Moses Turner * @ingroup aux_util */ @@ -32,6 +34,22 @@ enum u_device_alloc_flags // clang-format on }; +/*! + * + * Info to describe 2D extents of a device's screen + * + */ +struct u_extents_2d +{ + uint32_t w_pixels; // Width of entire screen in pixels + uint32_t h_pixels; // Height of entire screen +}; + +/*! + * + * Info to describe a very simple headset with diffractive lens optics. + * + */ struct u_device_simple_info { struct @@ -60,6 +78,17 @@ struct u_device_simple_info bool u_device_setup_split_side_by_side(struct xrt_device *xdev, const struct u_device_simple_info *info); +/*! + * Just setup the device's display's 2D extents. + * Good for headsets without traditional VR optics. + * + * @return true on success. + * @ingroup aux_util + */ +bool +u_extents_2d_split_side_by_side(struct xrt_device *xdev, const struct u_extents_2d *extents); + + /*! * Dump the device config to stderr. * @@ -103,7 +132,7 @@ void u_device_assign_xdev_roles(struct xrt_device **xdevs, size_t num_xdevs, int *head, int *left, int *right); /*! - * Helper function to assign head, left hand and right hand roles. + * Helper function for setting up tracking origins. Applies 3dof offsets for devices with XRT_TRACKING_TYPE_NONE. * * @ingroup aux_util */ @@ -113,6 +142,22 @@ u_device_setup_tracking_origins(struct xrt_device *head, struct xrt_device *right, struct xrt_vec3 *global_tracking_origin_offset); +/*! + * Helper function for `get_view_pose` in an HMD driver. + * + * Takes in a translation from the left to right eye, and returns a center to left or right eye transform that assumes + * the eye relation is symmetrical around the tracked point ("center eye"). Knowing IPD is a subset of this: If you know + * IPD better than the overall Monado system, copy @p eye_relation and put your known IPD in @p real_eye_relation->x + * + * If you have rotation, apply it after calling this function. + * + * @param eye_relation 3D translation from left eye to right eye. + * @param view_index 0 for left, 1 for right. + * @param out_pose The output pose to populate. Will receive translation, with an identity rotation. + */ +void +u_device_get_view_pose(const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose); + #ifdef __cplusplus } #endif diff --git a/src/xrt/auxiliary/util/u_distortion.c b/src/xrt/auxiliary/util/u_distortion.c index 1d4790f28..71384608e 100644 --- a/src/xrt/auxiliary/util/u_distortion.c +++ b/src/xrt/auxiliary/util/u_distortion.c @@ -29,8 +29,10 @@ u_distortion_cardboard_calculate(const struct u_cardboard_distortion_arguments * uint32_t h_pixels = args->screen.h_pixels; // Base assumption, the driver can change afterwards. - if (parts->blend_mode == 0) { - parts->blend_mode = XRT_BLEND_MODE_OPAQUE; + if (parts->num_blend_modes == 0) { + size_t idx = 0; + parts->blend_modes[idx++] = XRT_BLEND_MODE_OPAQUE; + parts->num_blend_modes = idx; } // Use the full screen. diff --git a/src/xrt/auxiliary/util/u_distortion_mesh.c b/src/xrt/auxiliary/util/u_distortion_mesh.c index 1374ccb4c..753765a15 100644 --- a/src/xrt/auxiliary/util/u_distortion_mesh.c +++ b/src/xrt/auxiliary/util/u_distortion_mesh.c @@ -1,9 +1,10 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Code to generate disortion meshes. * @author Jakob Bornecrantz + * @author Moses Turner * @ingroup aux_distortion */ @@ -14,6 +15,7 @@ #include "util/u_distortion_mesh.h" #include "math/m_vec2.h" +#include "math/m_api.h" #include #include @@ -266,6 +268,125 @@ u_compute_distortion_cardboard(struct u_cardboard_distortion_values *values, return true; } +/* + * + * North Star "2D Polynomial" distortion + * Sometimes known as "v2", filename is often NorthStarCalibration.json + * + */ + +static float +u_ns_polyval2d(float X, float Y, float C[16]) +{ + float X2 = X * X; + float X3 = X2 * X; + float Y2 = Y * Y; + float Y3 = Y2 * Y; + return (((C[0]) + (C[1] * Y) + (C[2] * Y2) + (C[3] * Y3)) + + ((C[4] * X) + (C[5] * X * Y) + (C[6] * X * Y2) + (C[7] * X * Y3)) + + ((C[8] * X2) + (C[9] * X2 * Y) + (C[10] * X2 * Y2) + (C[11] * X2 * Y3)) + + ((C[12] * X3) + (C[13] * X3 * Y) + (C[14] * X3 * Y2) + (C[15] * X3 * Y3))); +} + + +bool +u_compute_distortion_ns_p2d(struct u_ns_p2d_values *values, int view, float u, float v, struct xrt_uv_triplet *result) +{ + // I think that OpenCV and Monado have different definitions of v coordinates, but not sure. if not, + // unexplainable + v = 1.0 - v; + + float x_ray = u_ns_polyval2d(u, v, view ? values->x_coefficients_left : values->x_coefficients_right); + float y_ray = u_ns_polyval2d(u, v, view ? values->y_coefficients_left : values->y_coefficients_right); + + struct xrt_fov fov = values->fov[view]; + + float left_ray_bound = tan(fov.angle_left); + float right_ray_bound = tan(fov.angle_right); + float up_ray_bound = tan(fov.angle_up); + float down_ray_bound = tan(fov.angle_down); + + float u_eye = math_map_ranges(x_ray, left_ray_bound, right_ray_bound, 0, 1); + + float v_eye = math_map_ranges(y_ray, down_ray_bound, up_ray_bound, 0, 1); + + // boilerplate, put the UV coordinates in all the RGB slots + result->r.x = u_eye; + result->r.y = v_eye; + result->g.x = u_eye; + result->g.y = v_eye; + result->b.x = u_eye; + result->b.y = v_eye; + + return true; +} + + +/* + * + * Moses's "variable-IPD 2D" distortion + * If Moses goes away or stops using North Star for some reason, please remove this - as of june 2021 nobody else is + * using it. + * + */ + +bool +u_compute_distortion_ns_vipd(struct u_ns_vipd_values *values, int view, float u, float v, struct xrt_uv_triplet *result) +{ + int u_index_int = floorf(u * 64); + int v_index_int = floorf(v * 64); + float u_index_frac = (u * 64) - u_index_int; + float v_index_frac = (v * 64) - v_index_int; + + float x_ray; + float y_ray; + + if (u_index_frac > 0.0001) { + // Probably this codepath if grid size is not 65x65 + // {top,bottom}-{left,right} notation might be inaccurate. The code *works* right now but don't take its + // word when reading + struct xrt_vec2 topleft = values->grid_for_use.grid[view][v_index_int][u_index_int]; + struct xrt_vec2 topright = values->grid_for_use.grid[view][v_index_int][u_index_int + 1]; + struct xrt_vec2 bottomleft = values->grid_for_use.grid[view][v_index_int + 1][u_index_int]; + struct xrt_vec2 bottomright = values->grid_for_use.grid[view][v_index_int + 1][u_index_int + 1]; + struct xrt_vec2 leftcorrect = {math_map_ranges(v_index_frac, 0, 1, topleft.x, bottomleft.x), + math_map_ranges(v_index_frac, 0, 1, topleft.y, bottomleft.y)}; + struct xrt_vec2 rightcorrect = {math_map_ranges(v_index_frac, 0, 1, topright.x, bottomright.x), + math_map_ranges(v_index_frac, 0, 1, topright.y, bottomright.y)}; + y_ray = math_map_ranges(u_index_frac, 0, 1, leftcorrect.x, rightcorrect.x); + x_ray = math_map_ranges(u_index_frac, 0, 1, leftcorrect.y, rightcorrect.y); + } else { + // probably this path if grid size is 65x65 like normal + x_ray = values->grid_for_use.grid[view][v_index_int][u_index_int].y; + y_ray = values->grid_for_use.grid[view][v_index_int][u_index_int].x; + } + + struct xrt_fov fov = values->fov[view]; + + float left_ray_bound = tan(fov.angle_left); + float right_ray_bound = tan(fov.angle_right); + float up_ray_bound = tan(fov.angle_up); + float down_ray_bound = tan(fov.angle_down); + // printf("%f %f", fov.angle_down, fov.angle_up); + + float u_eye = math_map_ranges(x_ray, left_ray_bound, right_ray_bound, 0, 1); + + float v_eye = math_map_ranges(y_ray, down_ray_bound, up_ray_bound, 0, 1); + + // boilerplate, put the UV coordinates in all the RGB slots + result->r.x = u_eye; + result->r.y = v_eye; + result->g.x = u_eye; + result->g.y = v_eye; + result->b.x = u_eye; + result->b.y = v_eye; + // printf("%f %f\n", values->grid_for_use.grid[view][v_index_int][u_index_int].y, + // values->grid_for_use.grid[view][v_index_int][u_index_int].x); + + return true; +} + + bool u_compute_distortion_none(float u, float v, struct xrt_uv_triplet *result) { @@ -279,6 +400,7 @@ u_compute_distortion_none(float u, float v, struct xrt_uv_triplet *result) } + /* * * No distortion. diff --git a/src/xrt/auxiliary/util/u_distortion_mesh.h b/src/xrt/auxiliary/util/u_distortion_mesh.h index a878b6668..b67396a6b 100644 --- a/src/xrt/auxiliary/util/u_distortion_mesh.h +++ b/src/xrt/auxiliary/util/u_distortion_mesh.h @@ -4,6 +4,7 @@ * @file * @brief Code to generate disortion meshes. * @author Jakob Bornecrantz + * @author Moses Turner * @ingroup aux_distortion */ @@ -107,6 +108,60 @@ u_compute_distortion_cardboard(struct u_cardboard_distortion_values *values, struct xrt_uv_triplet *result); +/* + * + * North Star 2D/Polynomial distortion. + * + */ + +struct u_ns_p2d_values +{ + float x_coefficients_left[16]; + float x_coefficients_right[16]; + float y_coefficients_left[16]; + float y_coefficients_right[16]; + struct xrt_fov fov[2]; // left, right + float ipd; +}; + +/*! + * Distortion correction implementation for North Star 2D/Polynomial. + * + * @ingroup aux_distortion + */ +bool +u_compute_distortion_ns_p2d(struct u_ns_p2d_values *values, int view, float u, float v, struct xrt_uv_triplet *result); + +/* + * + * North Star 2D/"VIPD" distortion. + * + */ +struct u_ns_vipd_grid +{ + struct xrt_vec2 grid[2][65][65]; +}; + +struct u_ns_vipd_values +{ + int number_of_ipds; + float *ipds; + struct u_ns_vipd_grid *grids; + struct u_ns_vipd_grid grid_for_use; + struct xrt_fov fov[2]; // left, right + float ipd; +}; + +/*! + * Distortion correction implementation for North Star 2D/"VIPD". + * + * @ingroup aux_distortion + */ +bool +u_compute_distortion_ns_vipd( + struct u_ns_vipd_values *values, int view, float u, float v, struct xrt_uv_triplet *result); + + /* * * None distortion diff --git a/src/xrt/auxiliary/util/u_documentation.h b/src/xrt/auxiliary/util/u_documentation.h index 99dd6cd75..032892c8b 100644 --- a/src/xrt/auxiliary/util/u_documentation.h +++ b/src/xrt/auxiliary/util/u_documentation.h @@ -38,3 +38,29 @@ * * @brief Smaller pieces of auxiliary utilities code. */ + +#ifdef __cplusplus +/*! + * @brief C++-only APIs in Monado. + * + * There are not very many of them. + */ +namespace xrt { + +/*! + * @brief C++-only functionality from assorted helper libraries + */ +namespace auxiliary { + + /*! + * @brief C++-only functionality from the miscellaneous "util" helper library + */ + namespace util { + + } // namespace util + +} // namespace auxiliary + +} // namespace xrt + +#endif diff --git a/src/xrt/auxiliary/util/u_file.c b/src/xrt/auxiliary/util/u_file.c index ef90e6a07..43152e04f 100644 --- a/src/xrt/auxiliary/util/u_file.c +++ b/src/xrt/auxiliary/util/u_file.c @@ -54,10 +54,10 @@ mkpath(const char *path) ssize_t u_file_get_config_dir(char *out_path, size_t out_path_size) { - const char *xgd_home = getenv("XDG_CONFIG_HOME"); + const char *xdg_home = getenv("XDG_CONFIG_HOME"); const char *home = getenv("HOME"); - if (xgd_home != NULL) { - return snprintf(out_path, out_path_size, "%s/monado", xgd_home); + if (xdg_home != NULL) { + return snprintf(out_path, out_path_size, "%s/monado", xdg_home); } if (home != NULL) { return snprintf(out_path, out_path_size, "%s/.config/monado", home); @@ -104,4 +104,53 @@ u_file_open_file_in_config_dir(const char *filename, const char *mode) return fopen(file_str, mode); } +ssize_t +u_file_get_runtime_dir(char *out_path, size_t out_path_size) +{ + const char *xgd_rt = getenv("XDG_RUNTIME_DIR"); + if (xgd_rt != NULL) { + return snprintf(out_path, out_path_size, "%s", xgd_rt); + } + + const char *tmp = "/tmp"; + return snprintf(out_path, out_path_size, "%s", tmp); +} + +ssize_t +u_file_get_path_in_runtime_dir(const char *filename, char *out_path, size_t out_path_size) +{ + char tmp[PATH_MAX]; + ssize_t i = u_file_get_runtime_dir(tmp, sizeof(tmp)); + if (i <= 0) { + return -1; + } + + return snprintf(out_path, out_path_size, "%s/%s", tmp, filename); +} + #endif + +char * +u_file_read_content(FILE *file) +{ + // Go to the end of the file. + fseek(file, 0L, SEEK_END); + size_t file_size = ftell(file); + + // Return back to the start of the file. + fseek(file, 0L, SEEK_SET); + + char *buffer = (char *)calloc(file_size + 1, sizeof(char)); + if (buffer == NULL) { + return NULL; + } + + // Do the actual reading. + size_t ret = fread(buffer, sizeof(char), file_size, file); + if (ret != file_size) { + free(buffer); + return NULL; + } + + return buffer; +} diff --git a/src/xrt/auxiliary/util/u_file.cpp b/src/xrt/auxiliary/util/u_file.cpp new file mode 100644 index 000000000..4a2054fce --- /dev/null +++ b/src/xrt/auxiliary/util/u_file.cpp @@ -0,0 +1,94 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Very simple file opening functions, mostly using std::filesystem for portability. + * @author Ryan Pavlik + * @author Jakob Bornecrantz + * @author Pete Black + * @ingroup aux_util + */ + +#include "xrt/xrt_config_os.h" +#include "util/u_file.h" + +#ifndef XRT_OS_LINUX + +#include +#include +#include +#include + +#if __cplusplus >= 201703L +#include +namespace fs = std::filesystem; +#else +#define _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING +#include +namespace fs = std::experimental::filesystem; +#endif + +static inline fs::path +get_config_path() +{ +#ifdef XRT_OS_WINDOWS + auto local_app_data = fs::path{getenv("LOCALAPPDATA")}; + return local_app_data / "monado"; +#else + + const char *xdg_home = getenv("XDG_CONFIG_HOME"); + const char *home = getenv("HOME"); + if (xdg_home != NULL) { + return fs::path(xdg_home) / "monado"; + } + if (home != NULL) { + return fs::path(home) / "monado"; + } + return {}; +#endif +} +ssize_t +u_file_get_config_dir(char *out_path, size_t out_path_size) +{ + auto config_path = get_config_path(); + if (config_path.empty()) { + return -1; + } + auto config_path_string = config_path.string(); + return snprintf(out_path, out_path_size, "%s", config_path_string.c_str()); +} + +ssize_t +u_file_get_path_in_config_dir(const char *filename, char *out_path, size_t out_path_size) +{ + auto config_path = get_config_path(); + if (config_path.empty()) { + return -1; + } + auto path_string = (config_path / filename).string(); + return snprintf(out_path, out_path_size, "%s", path_string.c_str()); +} + +FILE * +u_file_open_file_in_config_dir(const char *filename, const char *mode) +{ + auto config_path = get_config_path(); + if (config_path.empty()) { + return NULL; + } + + auto file_path_string = (config_path / filename).string(); + FILE *file = fopen(file_path_string.c_str(), mode); + if (file != NULL) { + return file; + } + + // Try creating the path. + auto directory = (config_path / filename).parent_path(); + fs::create_directories(directory); + + // Do not report error. + return fopen(file_path_string.c_str(), mode); +} + +#endif diff --git a/src/xrt/auxiliary/util/u_file.h b/src/xrt/auxiliary/util/u_file.h index fddf662f0..fbbe57fb5 100644 --- a/src/xrt/auxiliary/util/u_file.h +++ b/src/xrt/auxiliary/util/u_file.h @@ -28,6 +28,14 @@ u_file_get_path_in_config_dir(const char *suffix, char *out_path, size_t out_pat FILE * u_file_open_file_in_config_dir(const char *filename, const char *mode); +ssize_t +u_file_get_runtime_dir(char *out_path, size_t out_path_size); + +char * +u_file_read_content(FILE *file); + +ssize_t +u_file_get_path_in_runtime_dir(const char *filename, char *out_path, size_t out_path_size); #ifdef __cplusplus } diff --git a/src/xrt/auxiliary/util/u_frame.c b/src/xrt/auxiliary/util/u_frame.c index 92337a1dd..5056f3870 100644 --- a/src/xrt/auxiliary/util/u_frame.c +++ b/src/xrt/auxiliary/util/u_frame.c @@ -42,3 +42,39 @@ u_frame_create_one_off(enum xrt_format f, uint32_t width, uint32_t height, struc xrt_frame_reference(out_frame, xf); } + +static void +free_clone(struct xrt_frame *xf) +{ + assert(xf->reference.count == 0); + free(xf->data); + free(xf); +} + +void +u_frame_clone(struct xrt_frame *to_copy, struct xrt_frame **out_frame) +{ + struct xrt_frame *xf = U_TYPED_CALLOC(struct xrt_frame); + + // Paranoia: Explicitly only copy the fields we want + xf->width = to_copy->width; + xf->height = to_copy->height; + xf->stride = to_copy->stride; + xf->size = to_copy->size; + + xf->format = to_copy->format; + xf->stereo_format = to_copy->stereo_format; + + xf->timestamp = to_copy->timestamp; + xf->source_timestamp = to_copy->source_timestamp; + xf->source_sequence = to_copy->source_sequence; + xf->source_id = to_copy->source_id; + + xf->destroy = free_clone; + + xf->data = malloc(xf->size); + + memcpy(xf->data, to_copy->data, xf->size); + + xrt_frame_reference(out_frame, xf); +} diff --git a/src/xrt/auxiliary/util/u_frame.h b/src/xrt/auxiliary/util/u_frame.h index 290aa03d0..3a95c09bb 100644 --- a/src/xrt/auxiliary/util/u_frame.h +++ b/src/xrt/auxiliary/util/u_frame.h @@ -23,6 +23,12 @@ extern "C" { void u_frame_create_one_off(enum xrt_format f, uint32_t width, uint32_t height, struct xrt_frame **out_frame); +/*! + * Clones a frame. The cloned frame is not freed when the original frame is freed; instead the cloned frame is freed + * when its reference reaches zero. + */ +void +u_frame_clone(struct xrt_frame *to_copy, struct xrt_frame **out_frame); #ifdef __cplusplus } diff --git a/src/xrt/auxiliary/util/u_generic_callbacks.hpp b/src/xrt/auxiliary/util/u_generic_callbacks.hpp new file mode 100644 index 000000000..f3d6c13f7 --- /dev/null +++ b/src/xrt/auxiliary/util/u_generic_callbacks.hpp @@ -0,0 +1,231 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Implementation of a generic callback collection, intended to be wrapped for a specific event type. + * @author Ryan Pavlik + * @ingroup aux_util + */ + +#include +#include + +namespace xrt::auxiliary::util { +template struct GenericCallbacks; + +namespace detail { + + /*! + * @brief Element type stored in @ref GenericCallbacks, for internal use only. + */ + template struct GenericCallbackEntry + { + CallbackType callback; + MaskType event_mask; + void *userdata; + bool should_remove = false; + + GenericCallbackEntry(CallbackType callback_, MaskType event_mask_, void *userdata_) noexcept + : callback(callback_), event_mask(event_mask_), userdata(userdata_) + {} + + /*! + * Do the two entries match? Used for removal "by value" + */ + bool + matches(GenericCallbackEntry const &other) const noexcept + { + return callback == other.callback && event_mask == other.event_mask && + userdata == other.userdata; + } + + bool + operator==(GenericCallbackEntry const &other) const noexcept + { + return matches(other); + } + + bool + shouldInvoke(MaskType event) const noexcept + { + return (event_mask & event) != 0; + } + }; + + template struct identity + { + using type = T; + }; + + // This lets us handle being passed an enum (which we can call underlying_type on) as well as an integer (which + // we cannot) + template + using mask_from_enum_t = + typename std::conditional_t::value, std::underlying_type, identity>::type; + +} // namespace detail + +/*! + * @brief A generic collection of callbacks for event types represented as a bitmask, intended to be wrapped for each + * usage. + * + * A registered callback may identify one or more event types (bits in the bitmask) that it wants to be invoked for. A + * userdata void pointer is also stored for each callback. Bitmasks are tested at invocation time, and the general + * callback format allows for callbacks to indicate they should be removed from the collection. Actually calling each + * callback is left to a consumer-provided "invoker" to allow adding context and event data to the call. The "invoker" + * also allows the option of whether or how to expose the self-removal capability: yours might simply always return + * "false". + * + * This generic structure supports callbacks that are included multiple times in the collection, if the consuming code + * needs it. GenericCallbacks::contains may be used by consuming code before conditionally calling addCallback, to + * limit to a single instance in a collection. + * + * @tparam CallbackType the function pointer type to store for each callback. + * @tparam EventType the event enum type. + */ +template struct GenericCallbacks +{ + +public: + static_assert(std::is_integral::value || std::is_enum::value, + "Your event type must either be an integer or an enum"); + using callback_t = CallbackType; + using event_t = EventType; + using mask_t = detail::mask_from_enum_t; + +private: + static_assert(std::is_integral::value, "Our enum to mask conversion should have produced an integer"); + + //! The type stored for each added callback. + using callback_entry_t = detail::GenericCallbackEntry; + +public: + /*! + * @brief Add a new callback entry with the given callback function pointer, event mask, and user data. + * + * New callback entries are always added at the end of the collection. + */ + void + addCallback(CallbackType callback, mask_t event_mask, void *userdata) + { + callbacks.emplace_back(callback, event_mask, userdata); + } + + /*! + * @brief Remove some number of callback entries matching the given callback function pointer, event mask, and + * user data. + * + * @param callback The callback function pointer. Tested for equality with each callback entry. + * @param event_mask The callback event mask. Tested for equality with each callback entry. + * @param userdata The opaque user data pointer. Tested for equality with each callback entry. + * @param num_skip The number of matches to skip before starting to remove callbacks. Defaults to 0. + * @param max_remove The number of matches to remove, or negative if no limit. Defaults to -1. + * + * @returns the number of callbacks removed. + */ + int + removeCallback( + CallbackType callback, mask_t event_mask, void *userdata, unsigned int num_skip = 0, int max_remove = -1) + { + if (max_remove == 0) { + // We were told to remove none. We can do this very quickly. + // Avoids a corner case in the loop where we assume max_remove is non-zero. + return 0; + } + bool found = false; + + const callback_entry_t needle{callback, event_mask, userdata}; + for (auto &entry : callbacks) { + if (entry.matches(needle)) { + if (num_skip > 0) { + // We are still in our skipping phase. + num_skip--; + continue; + } + entry.should_remove = true; + found = true; + // Negatives (no max) get more negative, which is OK. + max_remove--; + if (max_remove == 0) { + // not looking for more + break; + } + } + } + if (found) { + return purgeMarkedCallbacks(); + } + // if we didn't find any, we removed zero. + return 0; + } + + /*! + * @brief See if the collection contains at least one matching callback. + * + * @param callback The callback function pointer. Tested for equality with each callback entry. + * @param event_mask The callback event mask. Tested for equality with each callback entry. + * @param userdata The opaque user data pointer. Tested for equality with each callback entry. + * + * @returns true if a matching callback is found. + */ + bool + contains(CallbackType callback, mask_t event_mask, void *userdata) + { + const callback_entry_t needle{callback, event_mask, userdata}; + auto it = std::find(callbacks.begin(), callbacks.end(), needle); + return it != callbacks.end(); + } + + /*! + * @brief Invokes the callbacks, by passing the ones we should run to your "invoker" to add any desired + * context/event data and forward the call. + * + * Callbacks are called in order, filtering out those whose event mask does not include the given event. + * + * @param event The event type to invoke callbacks for. + * @param invoker A function/functor accepting the event, a callback function pointer, and the callback entry's + * userdata as parameters, and returning true if the callback should be removed from the collection. It is + * assumed that the invoker will add any additional context or event data and call the provided callback. + * + * Typically, a lambda with some captures and a single return statement will be sufficient for an invoker. + * + * @returns the number of callbacks run + */ + template + int + invokeCallbacks(EventType event, F &&invoker) + { + bool needPurge = false; + + int ran = 0; + for (auto &entry : callbacks) { + if (entry.shouldInvoke(static_cast(event))) { + bool willRemove = invoker(event, entry.callback, entry.userdata); + if (willRemove) { + entry.should_remove = true; + needPurge = true; + } + ran++; + } + } + if (needPurge) { + purgeMarkedCallbacks(); + } + return ran; + } + +private: + std::vector callbacks; + + int + purgeMarkedCallbacks() + { + auto b = callbacks.begin(); + auto e = callbacks.end(); + auto new_end = std::remove_if(b, e, [](callback_entry_t const &entry) { return entry.should_remove; }); + auto num_removed = std::distance(new_end, e); + callbacks.erase(new_end, e); + return static_cast(num_removed); + } +}; +} // namespace xrt::auxiliary::util diff --git a/src/xrt/auxiliary/util/u_hand_tracking.c b/src/xrt/auxiliary/util/u_hand_tracking.c index 8019f9b0c..2fde71145 100644 --- a/src/xrt/auxiliary/util/u_hand_tracking.c +++ b/src/xrt/auxiliary/util/u_hand_tracking.c @@ -60,7 +60,7 @@ struct u_joint_curl_model static struct u_joint_curl_model hand_joint_default_set_curl_model_defaults[XRT_HAND_JOINT_COUNT] = { // special cases: wrist and palm without bone lengths, offsets are // absolute, relative to hand origin (palm) - [XRT_HAND_JOINT_PALM] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_PALM] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0, .radius = 0.018, @@ -84,25 +84,25 @@ static struct u_joint_curl_model hand_joint_default_set_curl_model_defaults[XRT_ .radius = 0.015, .joint_id = XRT_HAND_JOINT_LITTLE_METACARPAL}, - [XRT_HAND_JOINT_LITTLE_PROXIMAL] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_LITTLE_PROXIMAL] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, DEG_TO_RAD(20), 0}, .bone_length = 0.035, .radius = 0.01, .joint_id = XRT_HAND_JOINT_LITTLE_PROXIMAL}, - [XRT_HAND_JOINT_LITTLE_INTERMEDIATE] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_LITTLE_INTERMEDIATE] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0.028, .radius = 0.009, .joint_id = XRT_HAND_JOINT_LITTLE_INTERMEDIATE}, - [XRT_HAND_JOINT_LITTLE_DISTAL] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_LITTLE_DISTAL] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0.022, .radius = 0.009, .joint_id = XRT_HAND_JOINT_LITTLE_DISTAL}, - [XRT_HAND_JOINT_LITTLE_TIP] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_LITTLE_TIP] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0, .radius = 0.012, @@ -115,25 +115,25 @@ static struct u_joint_curl_model hand_joint_default_set_curl_model_defaults[XRT_ .radius = 0.015, .joint_id = XRT_HAND_JOINT_RING_METACARPAL}, - [XRT_HAND_JOINT_RING_PROXIMAL] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_RING_PROXIMAL] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, DEG_TO_RAD(10), 0}, .bone_length = 0.040, .radius = 0.012, .joint_id = XRT_HAND_JOINT_RING_PROXIMAL}, - [XRT_HAND_JOINT_RING_INTERMEDIATE] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_RING_INTERMEDIATE] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0.031, .radius = 0.01, .joint_id = XRT_HAND_JOINT_RING_INTERMEDIATE}, - [XRT_HAND_JOINT_RING_DISTAL] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_RING_DISTAL] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0.023, .radius = 0.01, .joint_id = XRT_HAND_JOINT_RING_DISTAL}, - [XRT_HAND_JOINT_RING_TIP] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_RING_TIP] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0, .radius = 0.013, @@ -146,25 +146,25 @@ static struct u_joint_curl_model hand_joint_default_set_curl_model_defaults[XRT_ .radius = 0.012, .joint_id = XRT_HAND_JOINT_MIDDLE_METACARPAL}, - [XRT_HAND_JOINT_MIDDLE_PROXIMAL] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_MIDDLE_PROXIMAL] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0.042, .radius = 0.01, .joint_id = XRT_HAND_JOINT_MIDDLE_PROXIMAL}, - [XRT_HAND_JOINT_MIDDLE_INTERMEDIATE] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_MIDDLE_INTERMEDIATE] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0.033, .radius = 0.01, .joint_id = XRT_HAND_JOINT_MIDDLE_INTERMEDIATE}, - [XRT_HAND_JOINT_MIDDLE_DISTAL] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_MIDDLE_DISTAL] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0.024, .radius = 0.01, .joint_id = XRT_HAND_JOINT_MIDDLE_DISTAL}, - [XRT_HAND_JOINT_MIDDLE_TIP] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_MIDDLE_TIP] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0, .radius = 0.01, @@ -177,25 +177,25 @@ static struct u_joint_curl_model hand_joint_default_set_curl_model_defaults[XRT_ .radius = 0.012, .joint_id = XRT_HAND_JOINT_INDEX_METACARPAL}, - [XRT_HAND_JOINT_INDEX_PROXIMAL] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_INDEX_PROXIMAL] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, DEG_TO_RAD(-10), 0}, .bone_length = 0.040, .radius = 0.011, .joint_id = XRT_HAND_JOINT_INDEX_PROXIMAL}, - [XRT_HAND_JOINT_INDEX_INTERMEDIATE] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_INDEX_INTERMEDIATE] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0.031, .radius = 0.01, .joint_id = XRT_HAND_JOINT_INDEX_INTERMEDIATE}, - [XRT_HAND_JOINT_INDEX_DISTAL] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_INDEX_DISTAL] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0.023, .radius = 0.01, .joint_id = XRT_HAND_JOINT_INDEX_DISTAL}, - [XRT_HAND_JOINT_INDEX_TIP] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_INDEX_TIP] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0, .radius = 0.01, @@ -208,20 +208,20 @@ static struct u_joint_curl_model hand_joint_default_set_curl_model_defaults[XRT_ .radius = 0.0175, .joint_id = XRT_HAND_JOINT_THUMB_METACARPAL}, - [XRT_HAND_JOINT_THUMB_PROXIMAL] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_THUMB_PROXIMAL] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, DEG_TO_RAD(-12), 0}, .bone_length = 0.038, .radius = 0.017, .joint_id = XRT_HAND_JOINT_THUMB_PROXIMAL}, // no intermediate - [XRT_HAND_JOINT_THUMB_DISTAL] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_THUMB_DISTAL] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0.028, .radius = 0.017, .joint_id = XRT_HAND_JOINT_THUMB_DISTAL}, - [XRT_HAND_JOINT_THUMB_TIP] = {.position_offset = {.x = 0, .y = 0, .z = 0}, + [XRT_HAND_JOINT_THUMB_TIP] = {.position_offset = XRT_VEC3_ZERO, .axis_angle_offset = {0, 0, 0}, .bone_length = 0, .radius = 0.016, @@ -288,8 +288,8 @@ scale_model_param(struct u_joint_curl_model *param, float scale) } void -u_hand_joint_compute_next_by_curl(struct u_hand_tracking *set, - struct u_joint_space_relation *prev, +u_hand_joint_compute_next_by_curl(const struct u_hand_tracking *set, + const struct u_joint_space_relation *prev, enum xrt_hand hand, uint64_t at_timestamp_ns, struct u_joint_space_relation *out_joint, @@ -303,9 +303,6 @@ u_hand_joint_compute_next_by_curl(struct u_hand_tracking *set, scale_model_param(&prev_defaults, set->scale); scale_model_param(&defaults, set->scale); - struct xrt_vec3 x_axis = {1, 0, 0}; - struct xrt_vec3 y_axis = {0, 1, 0}; - // prev joint pose is transformed to next joint pose by adding the bone // vector to the joint, and adding rotation based on finger curl struct xrt_pose pose = prev->relation.pose; @@ -325,7 +322,8 @@ u_hand_joint_compute_next_by_curl(struct u_hand_tracking *set, //! @todo more axis rotations & make sure order is right //! @todo handle velocities - struct xrt_pose offset_pose; + const struct xrt_vec3 y_axis = XRT_VEC3_UNIT_Y; + struct xrt_pose offset_pose = XRT_POSE_IDENTITY; if (hand == XRT_HAND_LEFT) { quat_from_angle_vector_clockwise(defaults.axis_angle_offset[1], &y_axis, &offset_pose.orientation); offset_pose.position = defaults.position_offset; @@ -366,6 +364,7 @@ u_hand_joint_compute_next_by_curl(struct u_hand_tracking *set, float curl_angle = curl_value * full_curl_angle; + const struct xrt_vec3 x_axis = XRT_VEC3_UNIT_X; struct xrt_quat curl_rotation; math_quat_from_angle_vector(-curl_angle, &x_axis, &curl_rotation); math_quat_rotate(&pose.orientation, &curl_rotation, &pose.orientation); @@ -385,7 +384,7 @@ u_hand_joint_compute_next_by_curl(struct u_hand_tracking *set, math_quat_finite_difference(&old_relation.pose.orientation, &out_joint->relation.pose.orientation, time_diff_s, &out_joint->relation.angular_velocity); } else { - out_joint->relation.angular_velocity = (struct xrt_vec3){0, 0, 0}; + out_joint->relation.angular_velocity = (struct xrt_vec3)XRT_VEC3_ZERO; } out_joint->relation.relation_flags = POSE_VALID_FLAGS | VELOCIY_VALID_FLAGS; @@ -403,7 +402,7 @@ u_hand_joints_update_curl(struct u_hand_tracking *set, float curl_index = curl_values->index; float curl_thumb = curl_values->thumb; - const struct xrt_quat identity_quat = {0, 0, 0, 1}; + const struct xrt_quat identity_quat = XRT_QUAT_IDENTITY; //! @todo: full relations with velocities @@ -411,15 +410,15 @@ u_hand_joints_update_curl(struct u_hand_tracking *set, set->joints.wrist.relation.pose = (struct xrt_pose){ .position = hand_joint_default_set_curl_model_defaults[XRT_HAND_JOINT_WRIST].position_offset, .orientation = identity_quat}; - set->joints.wrist.relation.linear_velocity = (struct xrt_vec3){0, 0, 0}; - set->joints.wrist.relation.angular_velocity = (struct xrt_vec3){0, 0, 0}; + set->joints.wrist.relation.linear_velocity = (struct xrt_vec3)XRT_VEC3_ZERO; + set->joints.wrist.relation.angular_velocity = (struct xrt_vec3)XRT_VEC3_ZERO; set->joints.wrist.relation.relation_flags = POSE_VALID_FLAGS | VELOCIY_VALID_FLAGS; set->joints.palm.relation.pose = (struct xrt_pose){ .position = hand_joint_default_set_curl_model_defaults[XRT_HAND_JOINT_PALM].position_offset, .orientation = identity_quat}; - set->joints.palm.relation.linear_velocity = (struct xrt_vec3){0, 0, 0}; - set->joints.palm.relation.angular_velocity = (struct xrt_vec3){0, 0, 0}; + set->joints.palm.relation.linear_velocity = (struct xrt_vec3)XRT_VEC3_ZERO; + set->joints.palm.relation.angular_velocity = (struct xrt_vec3)XRT_VEC3_ZERO; set->joints.palm.relation.relation_flags = POSE_VALID_FLAGS | VELOCIY_VALID_FLAGS; struct u_joint_space_relation *prev = &set->joints.wrist; @@ -650,8 +649,8 @@ get_joint_data(struct u_hand_tracking *set, enum xrt_hand_joint joint_id) void u_hand_joints_set_out_data(struct u_hand_tracking *set, enum xrt_hand hand, - struct xrt_space_relation *hand_relation, - struct xrt_pose *hand_offset, + const struct xrt_space_relation *hand_relation, + const struct xrt_pose *hand_offset, struct xrt_hand_joint_set *out_value) { @@ -667,13 +666,19 @@ u_hand_joints_set_out_data(struct u_hand_tracking *set, m_space_graph_add_relation(&graph, &data->relation); m_space_graph_add_pose(&graph, hand_offset); m_space_graph_resolve(&graph, &l[i].relation); + + // joint relations can not be "more valid" than the hand relation + // after space graph to make sure flags are not "upgraded" + l[i].relation.relation_flags &= hand_relation->relation_flags; } out_value->hand_pose = *hand_relation; } void -u_hand_joints_offset_valve_index_controller(enum xrt_hand hand, struct xrt_vec3 *static_offset, struct xrt_pose *offset) +u_hand_joints_offset_valve_index_controller(enum xrt_hand hand, + const struct xrt_vec3 *static_offset, + struct xrt_pose *offset) { /* Controller space origin is at the very tip of the controller, * handle pointing forward at -z. @@ -686,9 +691,9 @@ u_hand_joints_offset_valve_index_controller(enum xrt_hand hand, struct xrt_vec3 * * Now the hand points "through the strap" like at normal use. */ - struct xrt_vec3 x = {1, 0, 0}; - struct xrt_vec3 y = {0, 1, 0}; - struct xrt_vec3 z = {0, 0, -1}; + const struct xrt_vec3 x = XRT_VEC3_UNIT_X; + const struct xrt_vec3 y = XRT_VEC3_UNIT_Y; + const struct xrt_vec3 negative_z = {0, 0, -1}; float hand_on_handle_x_rotation = DEG_TO_RAD(-72); float hand_on_handle_y_rotation = 0; @@ -700,13 +705,13 @@ u_hand_joints_offset_valve_index_controller(enum xrt_hand hand, struct xrt_vec3 } - struct xrt_quat hand_rotation_y = {0, 0, 0, 1}; + struct xrt_quat hand_rotation_y = XRT_QUAT_IDENTITY; math_quat_from_angle_vector(hand_on_handle_y_rotation, &y, &hand_rotation_y); - struct xrt_quat hand_rotation_z = {0, 0, 0, 1}; - math_quat_from_angle_vector(hand_on_handle_z_rotation, &z, &hand_rotation_z); + struct xrt_quat hand_rotation_z = XRT_QUAT_IDENTITY; + math_quat_from_angle_vector(hand_on_handle_z_rotation, &negative_z, &hand_rotation_z); - struct xrt_quat hand_rotation_x = {0, 0, 0, 1}; + struct xrt_quat hand_rotation_x = XRT_QUAT_IDENTITY; math_quat_from_angle_vector(hand_on_handle_x_rotation, &x, &hand_rotation_x); struct xrt_quat hand_rotation; diff --git a/src/xrt/auxiliary/util/u_hand_tracking.h b/src/xrt/auxiliary/util/u_hand_tracking.h index e8f2253ee..6e2265c6b 100644 --- a/src/xrt/auxiliary/util/u_hand_tracking.h +++ b/src/xrt/auxiliary/util/u_hand_tracking.h @@ -164,8 +164,8 @@ u_hand_joints_init_default_set(struct u_hand_tracking *set, void u_hand_joints_set_out_data(struct u_hand_tracking *set, enum xrt_hand hand, - struct xrt_space_relation *hand_relation, - struct xrt_pose *hand_offset, + const struct xrt_space_relation *hand_relation, + const struct xrt_pose *hand_offset, struct xrt_hand_joint_set *out_value); @@ -179,8 +179,8 @@ u_hand_joints_set_out_data(struct u_hand_tracking *set, * @ingroup aux_util */ void -u_hand_joint_compute_next_by_curl(struct u_hand_tracking *set, - struct u_joint_space_relation *prev, +u_hand_joint_compute_next_by_curl(const struct u_hand_tracking *set, + const struct u_joint_space_relation *prev, enum xrt_hand hand, uint64_t at_timestamp_ns, struct u_joint_space_relation *out_joint, @@ -202,7 +202,7 @@ u_hand_joints_update_curl(struct u_hand_tracking *set, */ void u_hand_joints_offset_valve_index_controller(enum xrt_hand hand, - struct xrt_vec3 *static_offset, + const struct xrt_vec3 *static_offset, struct xrt_pose *offset); diff --git a/src/xrt/auxiliary/util/u_handles.c b/src/xrt/auxiliary/util/u_handles.c index c6ddcce5c..7c9d56e80 100644 --- a/src/xrt/auxiliary/util/u_handles.c +++ b/src/xrt/auxiliary/util/u_handles.c @@ -1,4 +1,4 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -11,6 +11,12 @@ #include "u_handles.h" +/* + * + * Graphics Buffer Handles + * + */ + #if defined(XRT_GRAPHICS_BUFFER_HANDLE_IS_AHARDWAREBUFFER) #include @@ -89,3 +95,71 @@ u_graphics_buffer_unref(xrt_graphics_buffer_handle_t *handle_ptr) release_graphics_handle(handle); *handle_ptr = XRT_GRAPHICS_BUFFER_HANDLE_INVALID; } + +/* + * + * Graphics Sync Handles + * + */ + +#if defined(XRT_GRAPHICS_SYNC_HANDLE_IS_FD) +#include + +static inline void +release_sync_handle(xrt_graphics_sync_handle_t handle) +{ + close(handle); +} + +static inline xrt_graphics_sync_handle_t +ref_sync_handle(xrt_graphics_sync_handle_t handle) +{ + return dup(handle); +} + +#elif defined(XRT_GRAPHICS_SYNC_HANDLE_IS_WIN32_HANDLE) + +static inline void +release_sync_handle(xrt_graphics_sync_handle_t handle) +{ + CloseHandle(handle); +} + +static inline xrt_graphics_sync_handle_t +ref_sync_handle(xrt_graphics_sync_handle_t handle) +{ + HANDLE self = GetCurrentProcess(); + HANDLE result = NULL; + if (DuplicateHandle(self, handle, self, &result, 0, FALSE, DUPLICATE_SAME_ACCESS) != 0) { + return result; + } + return NULL; +} + +#else +#error "need port" +#endif + +xrt_graphics_sync_handle_t +u_graphics_sync_ref(xrt_graphics_sync_handle_t handle) +{ + if (xrt_graphics_sync_handle_is_valid(handle)) { + return ref_sync_handle(handle); + } + + return XRT_GRAPHICS_SYNC_HANDLE_INVALID; +} + +void +u_graphics_sync_unref(xrt_graphics_sync_handle_t *handle_ptr) +{ + if (handle_ptr == NULL) { + return; + } + xrt_graphics_sync_handle_t handle = *handle_ptr; + if (!xrt_graphics_sync_handle_is_valid(handle)) { + return; + } + release_sync_handle(handle); + *handle_ptr = XRT_GRAPHICS_SYNC_HANDLE_INVALID; +} diff --git a/src/xrt/auxiliary/util/u_handles.h b/src/xrt/auxiliary/util/u_handles.h index 34e7d9dad..a64d25cbf 100644 --- a/src/xrt/auxiliary/util/u_handles.h +++ b/src/xrt/auxiliary/util/u_handles.h @@ -1,4 +1,4 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-20211, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -17,13 +17,15 @@ #ifdef __cplusplus extern "C" { #endif + /*! * Increase the reference count on the buffer handle, returning the new * reference. * - * Depending on the underlying type, the value may be the same or different than - * what was passed in. It should be retained for use at release time, - * regardless. + * + * Depending on the underlying type, the value may be the same or different than what was passed in. It should be + * retained for use at release time, regardless. (For example, if the underlying native handle does not expose reference + * counting, it may be duplicated and the duplicate returned.) * * @public @memberof xrt_graphics_buffer_handle_t */ @@ -42,6 +44,31 @@ u_graphics_buffer_ref(xrt_graphics_buffer_handle_t handle); void u_graphics_buffer_unref(xrt_graphics_buffer_handle_t *handle); +/*! + * Increase the reference count on the sync handle, returning the new + * reference. + * + * Depending on the underlying type, the value may be the same or different than what was passed in. It should be + * retained for use at release time, regardless. (For example, if the underlying native handle does not expose reference + * counting, it may be duplicated and the duplicate returned.) + * + * @public @memberof xrt_graphics_sync_handle_t + */ +xrt_graphics_sync_handle_t +u_graphics_sync_ref(xrt_graphics_sync_handle_t handle); + +/*! + * Decrease the reference count/release the handle reference passed in. + * + * Be sure to only call this once per handle. + * + * Performs null-check and clears the value after unreferencing. + * + * @public @memberof xrt_graphics_sync_handle_t + */ +void +u_graphics_sync_unref(xrt_graphics_sync_handle_t *handle); + #ifdef __cplusplus } // extern "C" #endif diff --git a/src/xrt/auxiliary/util/u_json.c b/src/xrt/auxiliary/util/u_json.c index a8a595cc8..7b3d48240 100644 --- a/src/xrt/auxiliary/util/u_json.c +++ b/src/xrt/auxiliary/util/u_json.c @@ -21,6 +21,9 @@ #include #ifndef XRT_HAVE_SYSTEM_CJSON +#if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif // This includes the c file completely. #include "cjson/cJSON.c" #endif @@ -192,6 +195,42 @@ u_json_get_vec3_array(const cJSON *json, struct xrt_vec3 *out_vec3) return true; } +bool +u_json_get_vec3_f64_array(const cJSON *json, struct xrt_vec3_f64 *out_vec3) +{ + assert(out_vec3 != NULL); + + if (!json) { + return false; + } + if (!cJSON_IsArray(json)) { + return false; + } + + if (cJSON_GetArraySize(json) != 3) { + return false; + } + + double array[3] = {0, 0, 0}; + const cJSON *item = NULL; + size_t i = 0; + cJSON_ArrayForEach(item, json) + { + assert(cJSON_IsNumber(item)); + array[i] = item->valuedouble; + ++i; + if (i == 3) { + break; + } + } + + out_vec3->x = array[0]; + out_vec3->y = array[1]; + out_vec3->z = array[2]; + + return true; +} + bool u_json_get_quat(const cJSON *json, struct xrt_quat *out_quat) { diff --git a/src/xrt/auxiliary/util/u_json.h b/src/xrt/auxiliary/util/u_json.h index 55306e83b..8dc69ab66 100644 --- a/src/xrt/auxiliary/util/u_json.h +++ b/src/xrt/auxiliary/util/u_json.h @@ -70,7 +70,7 @@ bool u_json_get_double(const cJSON *json, double *out_double); /*! - * @brief Parse a vec3 from a JSON object. + * @brief Parse an xrt_vec3 from a JSON object. * * @return true if successful, false if not. */ @@ -78,13 +78,21 @@ bool u_json_get_vec3(const cJSON *json, struct xrt_vec3 *out_vec3); /*! - * @brief Parse a vec3 from a JSON array. + * @brief Parse an xrt_vec3 from a JSON array. * * @return true if successful, false if not. */ bool u_json_get_vec3_array(const cJSON *json, struct xrt_vec3 *out_vec3); +/*! + * @brief Parse an xrt_vec3_f64 from a JSON array. + * + * @return true if successful, false if not. + */ +bool +u_json_get_vec3_f64_array(const cJSON *json, struct xrt_vec3_f64 *out_vec3); + /*! * @brief Parse a quaternion from a JSON object. * diff --git a/src/xrt/auxiliary/util/u_logging.c b/src/xrt/auxiliary/util/u_logging.c index fb6f33c1c..89558af20 100644 --- a/src/xrt/auxiliary/util/u_logging.c +++ b/src/xrt/auxiliary/util/u_logging.c @@ -19,17 +19,10 @@ DEBUG_GET_ONCE_LOG_OPTION(global_log, "XRT_LOG", U_LOGGING_WARN) -enum u_logging_level global_log_level; - -static bool _is_log_level_initialized; - -void -_log_level_init() +enum u_logging_level +u_log_get_global_level(void) { - if (!_is_log_level_initialized) { - global_log_level = debug_get_log_option_global_log(); - _is_log_level_initialized = true; - } + return debug_get_log_option_global_log(); } #if defined(XRT_OS_ANDROID) @@ -50,10 +43,10 @@ u_log_convert_priority(enum u_logging_level level) } return ANDROID_LOG_INFO; } + void u_log(const char *file, int line, const char *func, enum u_logging_level level, const char *format, ...) { - _log_level_init(); // print_prefix(func, level); android_LogPriority prio = u_log_convert_priority(level); va_list args; @@ -71,7 +64,6 @@ u_log_xdev(const char *file, const char *format, ...) { - _log_level_init(); android_LogPriority prio = u_log_convert_priority(level); va_list args; va_start(args, format); @@ -107,7 +99,6 @@ print_prefix(int remainingBuf, char *buf, const char *func, enum u_logging_level void u_log(const char *file, int line, const char *func, enum u_logging_level level, const char *format, ...) { - _log_level_init(); char buf[16384] = {0}; @@ -131,7 +122,6 @@ u_log_xdev(const char *file, const char *format, ...) { - _log_level_init(); char buf[16384] = {0}; @@ -222,8 +212,6 @@ print_prefix(const char *func, enum u_logging_level level) void u_log(const char *file, int line, const char *func, enum u_logging_level level, const char *format, ...) { - _log_level_init(); - print_prefix(func, level); va_list args; @@ -243,8 +231,6 @@ u_log_xdev(const char *file, const char *format, ...) { - _log_level_init(); - print_prefix(func, level); va_list args; diff --git a/src/xrt/auxiliary/util/u_logging.h b/src/xrt/auxiliary/util/u_logging.h index dd9e46fbd..706284705 100644 --- a/src/xrt/auxiliary/util/u_logging.h +++ b/src/xrt/auxiliary/util/u_logging.h @@ -27,10 +27,23 @@ struct xrt_device; */ /*! - * @ingroup aux_log + * @addtogroup aux_log * @{ */ +/*! + * @brief Logging level enum + */ +enum u_logging_level +{ + U_LOGGING_TRACE, //!< Trace messages, highly verbose. + U_LOGGING_DEBUG, //!< Debug messages, verbose. + U_LOGGING_INFO, //!< Info messages: not very verbose, not indicating a problem. + U_LOGGING_WARN, //!< Warning messages: indicating a potential problem + U_LOGGING_ERROR, //!< Error messages: indicating a problem + U_LOGGING_RAW, //!< Special level for raw printing, prints a new-line. +}; + /*! * For places where you really just want printf, prints a new-line. */ @@ -39,23 +52,63 @@ struct xrt_device; u_log(__FILE__, __LINE__, __func__, U_LOGGING_RAW, __VA_ARGS__); \ } while (false) +/*! + * @name Base Logging Utilities + * In most cases, you will want to use another macro from this file, or a module/driver-local macro, to do your logging. + * @{ + */ +/*! + * @brief Log a message at @p level , with file, line, and function context (always logs) - typically wrapped + * in a helper macro. + * + * @param level A @ref u_logging_level value for this message. + * @param ... Format string and optional format arguments. + */ #define U_LOG(level, ...) \ do { \ u_log(__FILE__, __LINE__, __func__, level, __VA_ARGS__); \ } while (false) +/*! + * @brief Log at @p level only if the level is at least @p cond_level - typically wrapped in a helper macro. + * + * Adds file, line, and function context. Like U_LOG() but conditional. + * + * @param level A @ref u_logging_level value for this message. + * @param cond_level The minimum @ref u_logging_level that will be actually output. + * @param ... Format string and optional format arguments. + */ #define U_LOG_IFL(level, cond_level, ...) \ do { \ if (cond_level <= level) { \ u_log(__FILE__, __LINE__, __func__, level, __VA_ARGS__); \ } \ } while (false) - +/*! + * @brief Log at @p level for a given @ref xrt_device - typically wrapped in a helper macro. + * + * Adds file, line, and function context, and forwards device context from provided @p xdev . + * + * Like U_LOG() but calling u_log_xdev() (which takes a device) instead. + * + * @param level A @ref u_logging_level value for this message. + * @param xdev The @ref xrt_device pointer associated with this message. + * @param ... Format string and optional format arguments. + */ #define U_LOG_XDEV(level, xdev, ...) \ do { \ u_log_xdev(__FILE__, __LINE__, __func__, level, xdev, __VA_ARGS__); \ } while (false) - +/*! + * @brief Log at @p level for a given @ref xrt_device, only if the level is at least @p cond_level - typically wrapped + * in a helper macro. + * + * Adds file, line, and function context, and forwards device context from provided @p xdev . + * @param level A @ref u_logging_level value for this message. + * @param cond_level The minimum @ref u_logging_level that will be actually output. + * @param xdev The @ref xrt_device pointer associated with this message. + * @param ... Format string and optional format arguments. + */ #define U_LOG_XDEV_IFL(level, cond_level, xdev, ...) \ do { \ if (cond_level <= level) { \ @@ -63,48 +116,42 @@ struct xrt_device; } \ } while (false) -// clang-format off -#define U_LOG_T(...) U_LOG_IFL_T(global_log_level, __VA_ARGS__) -#define U_LOG_D(...) U_LOG_IFL_D(global_log_level, __VA_ARGS__) -#define U_LOG_I(...) U_LOG_IFL_I(global_log_level, __VA_ARGS__) -#define U_LOG_W(...) U_LOG_IFL_W(global_log_level, __VA_ARGS__) -#define U_LOG_E(...) U_LOG_IFL_E(global_log_level, __VA_ARGS__) -#define U_LOG_IFL_T(cond_level, ...) U_LOG_IFL(U_LOGGING_TRACE, cond_level, __VA_ARGS__) -#define U_LOG_IFL_D(cond_level, ...) U_LOG_IFL(U_LOGGING_DEBUG, cond_level, __VA_ARGS__) -#define U_LOG_IFL_I(cond_level, ...) U_LOG_IFL(U_LOGGING_INFO, cond_level, __VA_ARGS__) -#define U_LOG_IFL_W(cond_level, ...) U_LOG_IFL(U_LOGGING_WARN, cond_level, __VA_ARGS__) -#define U_LOG_IFL_E(cond_level, ...) U_LOG_IFL(U_LOGGING_ERROR, cond_level, __VA_ARGS__) - -#define U_LOG_XDEV_IFL_T(xdev, cond_level, ...) U_LOG_XDEV_IFL(U_LOGGING_TRACE, cond_level, xdev, __VA_ARGS__) -#define U_LOG_XDEV_IFL_D(xdev, cond_level, ...) U_LOG_XDEV_IFL(U_LOGGING_DEBUG, cond_level, xdev, __VA_ARGS__) -#define U_LOG_XDEV_IFL_I(xdev, cond_level, ...) U_LOG_XDEV_IFL(U_LOGGING_INFO, cond_level, xdev, __VA_ARGS__) -#define U_LOG_XDEV_IFL_W(xdev, cond_level, ...) U_LOG_XDEV_IFL(U_LOGGING_WARN, cond_level, xdev, __VA_ARGS__) -#define U_LOG_XDEV_IFL_E(xdev, cond_level, ...) U_LOG_XDEV_IFL(U_LOGGING_ERROR, cond_level, xdev, __VA_ARGS__) - -#define U_LOG_XDEV_T(xdev, ...) U_LOG_XDEV(U_LOGGING_TRACE, xdev, __VA_ARGS__) -#define U_LOG_XDEV_D(xdev, ...) U_LOG_XDEV(U_LOGGING_DEBUG, xdev, __VA_ARGS__) -#define U_LOG_XDEV_I(xdev, ...) U_LOG_XDEV(U_LOGGING_INFO, xdev, __VA_ARGS__) -#define U_LOG_XDEV_W(xdev, ...) U_LOG_XDEV(U_LOGGING_WARN, xdev, __VA_ARGS__) -#define U_LOG_XDEV_E(xdev, ...) U_LOG_XDEV(U_LOGGING_ERROR, xdev, __VA_ARGS__) -// clang-format on +/*! + * Returns the global logging level, subsystems own logging level take precedence. + */ enum u_logging_level -{ - U_LOGGING_TRACE, - U_LOGGING_DEBUG, - U_LOGGING_INFO, - U_LOGGING_WARN, - U_LOGGING_ERROR, - U_LOGGING_RAW, //!< Special level for raw printing, prints a new-line. -}; - -extern enum u_logging_level global_log_level; +u_log_get_global_level(void); +/*! + * @brief Main non-device-related log implementation function: do not call directly, use a macro that wraps it. + * + * This function always logs: level is used for printing or passed to native logging functions. + * + * @param file Source file name associated with a message + * @param line Source file line associated with a message + * @param func Function name associated with a message + * @param level Message level: used for formatting or forwarding to native log functions + * @param format Format string + * @param ... Format parameters + */ void u_log(const char *file, int line, const char *func, enum u_logging_level level, const char *format, ...) XRT_PRINTF_FORMAT(5, 6); +/*! + * @brief Main device-related log implementation function: do not call directly, use a macro that wraps it. + * + * This function always logs: level is used for printing or passed to native logging functions. + * @param file Source file name associated with a message + * @param line Source file line associated with a message + * @param func Function name associated with a message + * @param level Message level: used for formatting or forwarding to native log functions + * @param xdev The associated @ref xrt_device + * @param format Format string + * @param ... Format parameters + */ void u_log_xdev(const char *file, int line, @@ -114,6 +161,119 @@ u_log_xdev(const char *file, const char *format, ...) XRT_PRINTF_FORMAT(6, 7); +/*! + * @} + */ + + +/*! + * @name Logging macros conditional on global log level + * + * These each imply a log level, and will only log if the global log level is equal or lower. + * They are often used for one-off logging in a module with few other logging needs, + * where having a module-specific log level would be unnecessary. + * + * @see U_LOG_IFL, u_log_get_global_level() + * @param ... Format string and optional format arguments. + * @{ + */ +//! Log a message at U_LOGGING_TRACE level, conditional on the global log level +#define U_LOG_T(...) U_LOG_IFL_T(u_log_get_global_level(), __VA_ARGS__) + +//! Log a message at U_LOGGING_DEBUG level, conditional on the global log level +#define U_LOG_D(...) U_LOG_IFL_D(u_log_get_global_level(), __VA_ARGS__) + +//! Log a message at U_LOGGING_INFO level, conditional on the global log level +#define U_LOG_I(...) U_LOG_IFL_I(u_log_get_global_level(), __VA_ARGS__) + +//! Log a message at U_LOGGING_WARN level, conditional on the global log level +#define U_LOG_W(...) U_LOG_IFL_W(u_log_get_global_level(), __VA_ARGS__) + +//! Log a message at U_LOGGING_ERROR level, conditional on the global log level +#define U_LOG_E(...) U_LOG_IFL_E(u_log_get_global_level(), __VA_ARGS__) + +/*! + * @} + */ + +/*! + * @name Logging macros conditional on provided log level + * + * These are often wrapped within a module, to automatically supply + * @p cond_level as appropriate for that module. + * + * @see U_LOG_IFL + * @param cond_level The minimum @ref u_logging_level that will be actually output. + * @param ... Format string and optional format arguments. + * + * @{ + */ +//! Conditionally log a message at U_LOGGING_TRACE level. +#define U_LOG_IFL_T(cond_level, ...) U_LOG_IFL(U_LOGGING_TRACE, cond_level, __VA_ARGS__) +//! Conditionally log a message at U_LOGGING_DEBUG level. +#define U_LOG_IFL_D(cond_level, ...) U_LOG_IFL(U_LOGGING_DEBUG, cond_level, __VA_ARGS__) +//! Conditionally log a message at U_LOGGING_INFO level. +#define U_LOG_IFL_I(cond_level, ...) U_LOG_IFL(U_LOGGING_INFO, cond_level, __VA_ARGS__) +//! Conditionally log a message at U_LOGGING_WARN level. +#define U_LOG_IFL_W(cond_level, ...) U_LOG_IFL(U_LOGGING_WARN, cond_level, __VA_ARGS__) +//! Conditionally log a message at U_LOGGING_ERROR level. +#define U_LOG_IFL_E(cond_level, ...) U_LOG_IFL(U_LOGGING_ERROR, cond_level, __VA_ARGS__) +/*! + * @} + */ + + + +/*! + * @name Device-related logging macros conditional on provided log level + * + * These are often wrapped within a driver, to automatically supply @p xdev and + * @p cond_level from their conventional names and log level member variable. + * + * @param level A @ref u_logging_level value for this message. + * @param cond_level The minimum @ref u_logging_level that will be actually output. + * @param xdev The @ref xrt_device pointer associated with this message. + * @param ... Format string and optional format arguments. + * + * @{ + */ +//! Conditionally log a device-related message at U_LOGGING_TRACE level. +#define U_LOG_XDEV_IFL_T(xdev, cond_level, ...) U_LOG_XDEV_IFL(U_LOGGING_TRACE, cond_level, xdev, __VA_ARGS__) +//! Conditionally log a device-related message at U_LOGGING_DEBUG level. +#define U_LOG_XDEV_IFL_D(xdev, cond_level, ...) U_LOG_XDEV_IFL(U_LOGGING_DEBUG, cond_level, xdev, __VA_ARGS__) +//! Conditionally log a device-related message at U_LOGGING_INFO level. +#define U_LOG_XDEV_IFL_I(xdev, cond_level, ...) U_LOG_XDEV_IFL(U_LOGGING_INFO, cond_level, xdev, __VA_ARGS__) +//! Conditionally log a device-related message at U_LOGGING_WARN level. +#define U_LOG_XDEV_IFL_W(xdev, cond_level, ...) U_LOG_XDEV_IFL(U_LOGGING_WARN, cond_level, xdev, __VA_ARGS__) +//! Conditionally log a device-related message at U_LOGGING_ERROR level. +#define U_LOG_XDEV_IFL_E(xdev, cond_level, ...) U_LOG_XDEV_IFL(U_LOGGING_ERROR, cond_level, xdev, __VA_ARGS__) +/*! + * @} + */ + +/*! + * @name Device-related logging macros that always log. + * + * These wrap U_LOG_XDEV() to supply the @p level - which is only used for formatting the output, these macros always + * log regardless of level. + * + * @param xdev The @ref xrt_device pointer associated with this message. + * @param ... Format string and optional format arguments. + * @{ + */ +//! Log a device-related message at U_LOGGING_TRACE level (always logs). +#define U_LOG_XDEV_T(xdev, ...) U_LOG_XDEV(U_LOGGING_TRACE, xdev, __VA_ARGS__) +//! Log a device-related message at U_LOGGING_DEBUG level (always logs). +#define U_LOG_XDEV_D(xdev, ...) U_LOG_XDEV(U_LOGGING_DEBUG, xdev, __VA_ARGS__) +//! Log a device-related message at U_LOGGING_INFO level (always logs). +#define U_LOG_XDEV_I(xdev, ...) U_LOG_XDEV(U_LOGGING_INFO, xdev, __VA_ARGS__) +//! Log a device-related message at U_LOGGING_WARN level (always logs). +#define U_LOG_XDEV_W(xdev, ...) U_LOG_XDEV(U_LOGGING_WARN, xdev, __VA_ARGS__) +//! Log a device-related message at U_LOGGING_ERROR level (always logs). +#define U_LOG_XDEV_E(xdev, ...) U_LOG_XDEV(U_LOGGING_ERROR, xdev, __VA_ARGS__) +/*! + * @} + */ /*! * @} diff --git a/src/xrt/auxiliary/util/u_process.c b/src/xrt/auxiliary/util/u_process.c new file mode 100644 index 000000000..5e81624b2 --- /dev/null +++ b/src/xrt/auxiliary/util/u_process.c @@ -0,0 +1,105 @@ + +// Copyright 2019-2020, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Simple process handling + * @author Christoph Haag + * @ingroup aux_util + */ + +#include "xrt/xrt_config.h" + +#ifdef XRT_OS_LINUX + +#define PID_FILE_NAME "monado.pid" + +#ifdef XRT_HAVE_LIBBSD +#include +#endif + +#include + +#include +#include +#include "u_misc.h" +#include "u_file.h" +#include "u_logging.h" + +struct u_process +{ +#ifdef XRT_HAVE_LIBBSD + struct pidfh *pfh; +#else + int pid; +#endif +}; + +XRT_MAYBE_UNUSED static inline int +get_pidfile_path(char *buf) +{ + int size = u_file_get_path_in_runtime_dir(PID_FILE_NAME, buf, PATH_MAX); + if (size == -1) { + U_LOG_W("Failed to determine runtime dir, not creating pidfile"); + return -1; + } + return 0; +} + +struct u_process * +u_process_create_if_not_running() +{ +#ifdef XRT_HAVE_LIBBSD + + char tmp[PATH_MAX]; + if (get_pidfile_path(tmp) < 0) { + U_LOG_W("Failed to determine runtime dir, not creating pidfile"); + return NULL; + } + + U_LOG_T("Using pidfile %s", tmp); + + pid_t otherpid; + + struct pidfh *pfh = pidfile_open(tmp, 0600, &otherpid); + if (errno == EEXIST || pfh == NULL) { + U_LOG_T("Failed to create pidfile (%s): Another Monado instance may be running", strerror(errno)); + // other process is locking pid file + return NULL; + } + + // either new or stale pidfile opened + + int write_ret = pidfile_write(pfh); + if (write_ret != 0) { + pidfile_close(pfh); + return NULL; + } + + struct u_process *ret = U_TYPED_CALLOC(struct u_process); + ret->pfh = pfh; + + U_LOG_T("No other Monado instance was running, got new pidfile"); + return ret; +#else + struct u_process *ret = U_TYPED_CALLOC(struct u_process); + //! @todo alternative implementation + ret->pid = 0; + return ret; +#endif +} + +void +u_process_destroy(struct u_process *proc) +{ + if (proc == NULL) { + return; + } + +#ifdef XRT_HAVE_LIBBSD + pidfile_close(proc->pfh); +#endif + free(proc); +} + +#endif diff --git a/src/xrt/auxiliary/util/u_process.h b/src/xrt/auxiliary/util/u_process.h new file mode 100644 index 000000000..1bbde8c04 --- /dev/null +++ b/src/xrt/auxiliary/util/u_process.h @@ -0,0 +1,42 @@ +// Copyright 2020, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Simple process handling + * @author Christoph Haag + * @ingroup aux_util + */ + +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct u_process; + +/*! + * Creates a handle for this process that is unique to the operating system user. Returns NULL if another process + * holding a handle is already running. + * + * @todo If built without libbsd support, a dummy value is returned that needs to be handled by the caller. + * + * @return a new u_process handle if no monado instance is running, NULL if another instance is already running. + * @ingroup aux_util + */ +struct u_process * +u_process_create_if_not_running(); + +/*! + * Releases the unique handle of the operating system user. + * + * @ingroup aux_util + */ +void +u_process_destroy(struct u_process *proc); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/util/u_render_timing.c b/src/xrt/auxiliary/util/u_render_timing.c deleted file mode 100644 index 77e69dd66..000000000 --- a/src/xrt/auxiliary/util/u_render_timing.c +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2020, Collabora, Ltd. -// SPDX-License-Identifier: BSL-1.0 -/*! - * @file - * @brief Shared frame timing code. - * @author Jakob Bornecrantz - * @ingroup aux_util - */ - -#include "util/u_misc.h" -#include "util/u_logging.h" -#include "util/u_render_timing.h" - -#include -#include -#include - - -/* - * - * Helpers - * - */ - -#if 0 -#define DEBUG_PRINT_FRAME_ID() U_LOG_RAW("%" PRIi64 " %s", frame_id, __func__) -#else -#define DEBUG_PRINT_FRAME_ID() \ - do { \ - } while (false) -#endif - -static uint64_t -min_period(const struct u_rt_helper *urth) -{ - return urth->last_input.predicted_display_period_ns; -} - -static uint64_t -last_displayed(const struct u_rt_helper *urth) -{ - return urth->last_input.predicted_display_time_ns; -} - -static uint64_t -get_last_input_plus_period_at_least_greater_then(struct u_rt_helper *urth, uint64_t then_ns) -{ - uint64_t val = last_displayed(urth); - - if (min_period(urth) == 0) { - return then_ns; - } - - while (val <= then_ns) { - val += min_period(urth); - assert(val != 0); - } - - - return val; -} - - -/* - * - * 'Exported' functions. - * - */ - -void -u_rt_helper_client_clear(struct u_rt_helper *urth) -{ - for (size_t i = 0; i < ARRAY_SIZE(urth->frames); i++) { - urth->frames[i].state = U_RT_READY; - urth->frames[i].frame_id = -1; - } -} - -void -u_rt_helper_init(struct u_rt_helper *urth) -{ - U_ZERO(urth); - u_rt_helper_client_clear(urth); -} - -void -u_rt_helper_predict(struct u_rt_helper *urth, - int64_t *out_frame_id, - uint64_t *predicted_display_time, - uint64_t *wake_up_time, - uint64_t *predicted_display_period, - uint64_t *min_display_period) -{ - int64_t frame_id = ++urth->frame_counter; - *out_frame_id = frame_id; - - DEBUG_PRINT_FRAME_ID(); - - uint64_t at_least_ns = os_monotonic_get_ns(); - - // Don't return a time before the last returned type. - if (at_least_ns < urth->last_returned_ns) { - at_least_ns = urth->last_returned_ns; - } - - uint64_t predict_ns = get_last_input_plus_period_at_least_greater_then(urth, at_least_ns); - - urth->last_returned_ns = predict_ns; - - *wake_up_time = predict_ns - min_period(urth); - *predicted_display_time = predict_ns; - *predicted_display_period = min_period(urth); - *min_display_period = min_period(urth); - - size_t index = (uint64_t)frame_id % ARRAY_SIZE(urth->frames); - assert(urth->frames[index].frame_id == -1); - assert(urth->frames[index].state == U_RT_READY); - - urth->frames[index].predicted = os_monotonic_get_ns(); - urth->frames[index].state = U_RT_PREDICTED; - urth->frames[index].frame_id = frame_id; -} - -void -u_rt_helper_mark_wait_woke(struct u_rt_helper *urth, int64_t frame_id) -{ - DEBUG_PRINT_FRAME_ID(); - - size_t index = (uint64_t)frame_id % ARRAY_SIZE(urth->frames); - assert(urth->frames[index].frame_id == frame_id); - assert(urth->frames[index].state == U_RT_PREDICTED); - - urth->frames[index].wait_woke = os_monotonic_get_ns(); - urth->frames[index].state = U_RT_WAIT_LEFT; -} - -void -u_rt_helper_mark_begin(struct u_rt_helper *urth, int64_t frame_id) -{ - DEBUG_PRINT_FRAME_ID(); - - size_t index = (uint64_t)frame_id % ARRAY_SIZE(urth->frames); - assert(urth->frames[index].frame_id == frame_id); - assert(urth->frames[index].state == U_RT_WAIT_LEFT); - - urth->frames[index].begin = os_monotonic_get_ns(); - urth->frames[index].state = U_RT_BEGUN; -} - -void -u_rt_helper_mark_discarded(struct u_rt_helper *urth, int64_t frame_id) -{ - DEBUG_PRINT_FRAME_ID(); - - size_t index = (uint64_t)frame_id % ARRAY_SIZE(urth->frames); - assert(urth->frames[index].frame_id == frame_id); - assert(urth->frames[index].state == U_RT_WAIT_LEFT || urth->frames[index].state == U_RT_BEGUN); - - urth->frames[index].end_frame = os_monotonic_get_ns(); - urth->frames[index].state = U_RT_READY; - urth->frames[index].frame_id = -1; -} - -void -u_rt_helper_mark_delivered(struct u_rt_helper *urth, int64_t frame_id) -{ - DEBUG_PRINT_FRAME_ID(); - - size_t index = (uint64_t)frame_id % ARRAY_SIZE(urth->frames); - assert(urth->frames[index].frame_id == frame_id); - assert(urth->frames[index].state == U_RT_BEGUN); - - uint64_t now_ns = os_monotonic_get_ns(); - - urth->frames[index].end_frame = now_ns; - urth->frames[index].state = U_RT_READY; - urth->frames[index].frame_id = -1; - -#if 0 - uint64_t then_ns = urth->frames[index].wait_woke; - uint64_t diff_ns = now_ns - then_ns; - uint64_t ms100 = diff_ns / (1000 * 10); - - U_LOG_RAW("Diff %i.%02ims", (int)ms100 / 100, (int)ms100 % 100); -#endif -} - -void -u_rt_helper_new_sample(struct u_rt_helper *urth, - uint64_t predicted_display_time_ns, - uint64_t predicted_display_period_ns, - uint64_t extra_ns) -{ - urth->last_input.predicted_display_time_ns = predicted_display_time_ns; - urth->last_input.predicted_display_period_ns = predicted_display_period_ns; - urth->last_input.extra_ns = extra_ns; -} diff --git a/src/xrt/auxiliary/util/u_render_timing.h b/src/xrt/auxiliary/util/u_render_timing.h deleted file mode 100644 index c7629ec11..000000000 --- a/src/xrt/auxiliary/util/u_render_timing.h +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2020, Collabora, Ltd. -// SPDX-License-Identifier: BSL-1.0 -/*! - * @file - * @brief Shared frame timing code. - * @author Jakob Bornecrantz - * @ingroup aux_util - */ - -#pragma once - -#include "xrt/xrt_compiler.h" -#include "os/os_time.h" - -#ifdef __cplusplus -extern "C" { -#endif - - -enum u_rt_state -{ - U_RT_READY, - U_RT_WAIT_LEFT, - U_RT_PREDICTED, - U_RT_BEGUN, -}; - -struct u_rt_frame -{ - uint64_t predicted; - uint64_t wait_woke; - uint64_t begin; - uint64_t end_frame; - int64_t frame_id; - enum u_rt_state state; -}; - -/*! - * This render timing helper is designed to schedule the rendering time of - * clients that submit frames to a compositor, which runs its own render loop - * that picks latest completed frames for that client. - */ -struct u_rt_helper -{ - struct u_rt_frame frames[2]; - uint32_t current_frame; - uint32_t next_frame; - - int64_t frame_counter; - - struct - { - //! The last display time that the thing driving this helper got. - uint64_t predicted_display_time_ns; - //! The last display period the hardware is running at. - uint64_t predicted_display_period_ns; - //! The extra time needed by the thing driving this helper. - uint64_t extra_ns; - } last_input; - - uint64_t last_returned_ns; -}; - -void -u_rt_helper_init(struct u_rt_helper *urth); - -/*! - * This function gets the client part of the render timing helper ready to be - * used. If you use init you will also clear all of the timing information. - * - * Call this when resetting a client. - */ -void -u_rt_helper_client_clear(struct u_rt_helper *urth); - -/*! - * Predict when the client's next rendered frame will be presented, also when - * the client should be woken up from sleeping, its display period and the - * minimum display period that the client might have. - * - * This is called from `xrWaitFrame`, but it does not do any waiting, the caller - * should wait till `out_wake_up_time`. - */ -void -u_rt_helper_predict(struct u_rt_helper *urth, - int64_t *out_frame_id, - uint64_t *out_predicted_display_time, - uint64_t *out_wake_up_time, - uint64_t *out_predicted_display_period, - uint64_t *out_min_display_period); - -/*! - * Log when the client woke up after sleeping for the time returned in - * @ref u_rt_helper_predict. This happens inside of `xrWaitFrame`. - */ -void -u_rt_helper_mark_wait_woke(struct u_rt_helper *urth, int64_t frame_id); - -/*! - * The client has started rendering work, see `xrBeginFrame`. - */ -void -u_rt_helper_mark_begin(struct u_rt_helper *urth, int64_t frame_id); - -/*! - * When a frame has been discarded. - */ -void -u_rt_helper_mark_discarded(struct u_rt_helper *urth, int64_t frame_id); - -/*! - * A frame has been delivered from the client, see `xrEndFrame`. The GPU might - * still be rendering the work. - */ -void -u_rt_helper_mark_delivered(struct u_rt_helper *urth, int64_t frame_id); - -/*! - * Add a new sample point from the main render loop. - * - * This is called in the main renderer loop that tightly submits frames to the - * real compositor for displaying. This is only used to inform the render helper - * when the frame will be shown, not any timing information about the client. - * - * When this is called doesn't matter that much, as the render timing will need - * to be able to predict one or more frames into the future anyways. But - * preferably as soon as the main loop wakes up from wait frame. - * - * @param urth Self pointer - * @param predicted_display_time_ns Predicted display time for this sample. - * @param predicted_display_period_ns Predicted display period for this sample. - * @param extra_ns Time between display and when this sample - * was created, that is when the main loop - * was woken up by the main compositor. - */ -void -u_rt_helper_new_sample(struct u_rt_helper *urth, - uint64_t predicted_display_time_ns, - uint64_t predicted_display_period_ns, - uint64_t extra_ns); - - -#ifdef __cplusplus -} -#endif diff --git a/src/xrt/auxiliary/util/u_sink.h b/src/xrt/auxiliary/util/u_sink.h index 44558fc74..65af7e25b 100644 --- a/src/xrt/auxiliary/util/u_sink.h +++ b/src/xrt/auxiliary/util/u_sink.h @@ -9,8 +9,10 @@ #pragma once +#include "os/os_threading.h" #include "xrt/xrt_frame.h" + #ifdef __cplusplus extern "C" { #endif @@ -26,8 +28,8 @@ struct u_sink_quirk_params }; /*! - * @relatesalso xrt_frame_sink - * @relates xrt_frame_context + * @public @memberof xrt_frame_sink + * @see xrt_frame_context */ void u_sink_create_format_converter(struct xrt_frame_context *xfctx, @@ -36,8 +38,8 @@ u_sink_create_format_converter(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs); /*! - * @relatesalso xrt_frame_sink - * @relates xrt_frame_context + * @public @memberof xrt_frame_sink + * @see xrt_frame_context */ void u_sink_create_to_r8g8b8_or_l8(struct xrt_frame_context *xfctx, @@ -45,8 +47,8 @@ u_sink_create_to_r8g8b8_or_l8(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs); /*! - * @relatesalso xrt_frame_sink - * @relates xrt_frame_context + * @public @memberof xrt_frame_sink + * @see xrt_frame_context */ void u_sink_create_to_r8g8b8_bayer_or_l8(struct xrt_frame_context *xfctx, @@ -54,8 +56,17 @@ u_sink_create_to_r8g8b8_bayer_or_l8(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs); /*! - * @relatesalso xrt_frame_sink - * @relates xrt_frame_context + * @public @memberof xrt_frame_sink + * @see xrt_frame_context + */ +void +u_sink_create_to_rgb_yuv_yuyv_uyvy_or_l8(struct xrt_frame_context *xfctx, + struct xrt_frame_sink *downstream, + struct xrt_frame_sink **out_xfs); + +/*! + * @public @memberof xrt_frame_sink + * @see xrt_frame_context */ void u_sink_create_to_yuv_yuyv_uyvy_or_l8(struct xrt_frame_context *xfctx, @@ -63,17 +74,16 @@ u_sink_create_to_yuv_yuyv_uyvy_or_l8(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs); /*! - * @relatesalso xrt_frame_sink - * @relates xrt_frame_context + * @public @memberof xrt_frame_sink + * @see xrt_frame_context */ void u_sink_create_to_yuv_or_yuyv(struct xrt_frame_context *xfctx, struct xrt_frame_sink *downstream, struct xrt_frame_sink **out_xfs); /*! - * @public @memberof u_sink_deinterleaver - * @relatesalso xrt_frame_sink - * @relates xrt_frame_context + * @public @memberof xrt_frame_sink + * @see xrt_frame_context */ void u_sink_deinterleaver_create(struct xrt_frame_context *xfctx, @@ -81,9 +91,8 @@ u_sink_deinterleaver_create(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs); /*! - * @public @memberof u_sink_queue - * @relatesalso xrt_frame_sink - * @relates xrt_frame_context + * @public @memberof xrt_frame_sink + * @see xrt_frame_context */ bool u_sink_queue_create(struct xrt_frame_context *xfctx, @@ -91,9 +100,8 @@ u_sink_queue_create(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs); /*! - * @public @memberof u_sink_quirk - * @relatesalso xrt_frame_sink - * @relates xrt_frame_context + * @public @memberof xrt_frame_sink + * @see xrt_frame_context */ void u_sink_quirk_create(struct xrt_frame_context *xfctx, @@ -102,8 +110,8 @@ u_sink_quirk_create(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs); /*! - * @public @memberof u_sink_split - * @relatesalso xrt_frame_sink + * @public @memberof xrt_frame_sink + * @see xrt_frame_context */ void u_sink_split_create(struct xrt_frame_context *xfctx, @@ -111,6 +119,66 @@ u_sink_split_create(struct xrt_frame_context *xfctx, struct xrt_frame_sink *right, struct xrt_frame_sink **out_xfs); + +/* + * + * Debugging sink, + * + */ + +/*! + * Allows more safely to debug sink inputs and outputs. + */ +struct u_sink_debug +{ + //! Is initialised/destroyed when added or root is removed. + struct os_mutex mutex; + + // Protected by mutex, mutex must be held when frame is being pushed. + struct xrt_frame_sink *sink; +}; + +static inline void +u_sink_debug_init(struct u_sink_debug *usd) +{ + os_mutex_init(&usd->mutex); +} + +static inline bool +u_sink_debug_is_active(struct u_sink_debug *usd) +{ + os_mutex_lock(&usd->mutex); + bool active = usd->sink != NULL; + os_mutex_unlock(&usd->mutex); + + return active; +} + +static inline void +u_sink_debug_push_frame(struct u_sink_debug *usd, struct xrt_frame *xf) +{ + os_mutex_lock(&usd->mutex); + if (usd->sink != NULL) { + xrt_sink_push_frame(usd->sink, xf); + } + os_mutex_unlock(&usd->mutex); +} + +static inline void +u_sink_debug_set_sink(struct u_sink_debug *usd, struct xrt_frame_sink *xfs) +{ + os_mutex_lock(&usd->mutex); + usd->sink = xfs; + os_mutex_unlock(&usd->mutex); +} + +static inline void +u_sink_debug_destroy(struct u_sink_debug *usd) +{ + os_mutex_destroy(&usd->mutex); +} + + #ifdef __cplusplus } #endif diff --git a/src/xrt/auxiliary/util/u_sink_converter.c b/src/xrt/auxiliary/util/u_sink_converter.c index 89f6b5e48..a0be73fda 100644 --- a/src/xrt/auxiliary/util/u_sink_converter.c +++ b/src/xrt/auxiliary/util/u_sink_converter.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -13,6 +13,7 @@ #include "util/u_sink.h" #include "util/u_frame.h" #include "util/u_format.h" +#include "util/u_trace_marker.h" #include @@ -44,6 +45,30 @@ struct u_sink_converter }; +/* + * + * L8 functions. + * + */ + +static void +from_L8_to_R8G8B8(struct xrt_frame *dst_frame, uint32_t w, uint32_t h, size_t stride, const uint8_t *data) +{ + SINK_TRACE_MARKER(); + + for (uint32_t y = 0; y < h; y++) { + for (uint32_t x = 0; x < w; x += 1) { + const uint8_t *src = data; + uint8_t *dst = dst_frame->data; + + src = src + (y * stride) + x; + dst = dst + (y * dst_frame->stride) + (x * 3); + dst[2] = dst[1] = dst[0] = src[0]; + } + } +} + + /* * * YUV functions. @@ -192,6 +217,8 @@ YUV444_to_R8G8B8(const uint8_t *input, uint8_t *dst) static void from_YUYV422_to_R8G8B8(struct xrt_frame *dst_frame, uint32_t w, uint32_t h, size_t stride, const uint8_t *data) { + SINK_TRACE_MARKER(); + for (uint32_t y = 0; y < h; y++) { for (uint32_t x = 0; x < w; x += 2) { const uint8_t *src = data; @@ -207,6 +234,8 @@ from_YUYV422_to_R8G8B8(struct xrt_frame *dst_frame, uint32_t w, uint32_t h, size static void from_UYVY422_to_R8G8B8(struct xrt_frame *dst_frame, uint32_t w, uint32_t h, size_t stride, const uint8_t *data) { + SINK_TRACE_MARKER(); + for (uint32_t y = 0; y < h; y++) { for (uint32_t x = 0; x < w; x += 2) { const uint8_t *src = data; @@ -223,6 +252,8 @@ from_UYVY422_to_R8G8B8(struct xrt_frame *dst_frame, uint32_t w, uint32_t h, size static void from_YUV888_to_R8G8B8(struct xrt_frame *dst_frame, uint32_t w, uint32_t h, size_t stride, const uint8_t *data) { + SINK_TRACE_MARKER(); + for (uint32_t y = 0; y < h; y++) { for (uint32_t x = 0; x < w; x++) { const uint8_t *src = data; @@ -262,6 +293,8 @@ check_header(size_t size, const uint8_t *data) static bool from_MJPEG_to_R8G8B8(struct xrt_frame *dst_frame, size_t size, const uint8_t *data) { + SINK_TRACE_MARKER(); + if (!check_header(size, data)) { return false; } @@ -302,6 +335,8 @@ from_MJPEG_to_R8G8B8(struct xrt_frame *dst_frame, size_t size, const uint8_t *da static bool from_MJPEG_to_YUV888(struct xrt_frame *dst_frame, size_t size, const uint8_t *data) { + SINK_TRACE_MARKER(); + if (!check_header(size, data)) { return false; } @@ -350,6 +385,8 @@ from_MJPEG_to_YUV888(struct xrt_frame *dst_frame, size_t size, const uint8_t *da static void from_BAYER_GR8_to_R8G8B8(struct xrt_frame *dst_frame, uint32_t w, uint32_t h, size_t stride, const uint8_t *data) { + SINK_TRACE_MARKER(); + const uint8_t *src_data = data; uint32_t src_stride = stride; @@ -424,8 +461,10 @@ create_frame_with_format(struct xrt_frame *xf, enum xrt_format format, struct xr } static void -receive_frame_r8g8b8_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf) +convert_frame_r8g8b8_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf) { + SINK_TRACE_MARKER(); + struct u_sink_converter *s = (struct u_sink_converter *)xs; struct xrt_frame *converted = NULL; @@ -465,6 +504,8 @@ receive_frame_r8g8b8_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf) return; } if (!from_MJPEG_to_R8G8B8(converted, xf->size, xf->data)) { + // Make sure to free frame when we fail to decode. + xrt_frame_reference(&converted, NULL); return; } break; @@ -479,8 +520,10 @@ receive_frame_r8g8b8_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf) } static void -receive_frame_r8g8b8_bayer_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf) +convert_frame_r8g8b8_bayer_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf) { + SINK_TRACE_MARKER(); + struct u_sink_converter *s = (struct u_sink_converter *)xs; struct xrt_frame *converted = NULL; @@ -513,6 +556,8 @@ receive_frame_r8g8b8_bayer_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf return; } if (!from_MJPEG_to_R8G8B8(converted, xf->size, xf->data)) { + // Make sure to free frame when we fail to decode. + xrt_frame_reference(&converted, NULL); return; } break; @@ -527,14 +572,22 @@ receive_frame_r8g8b8_bayer_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf } static void -receive_frame_r8g8b8(struct xrt_frame_sink *xs, struct xrt_frame *xf) +convert_frame_r8g8b8(struct xrt_frame_sink *xs, struct xrt_frame *xf) { + SINK_TRACE_MARKER(); + struct u_sink_converter *s = (struct u_sink_converter *)xs; struct xrt_frame *converted = NULL; switch (xf->format) { case XRT_FORMAT_R8G8B8: s->downstream->push_frame(s->downstream, xf); return; + case XRT_FORMAT_L8: + if (!create_frame_with_format(xf, XRT_FORMAT_R8G8B8, &converted)) { + return; + } + from_L8_to_R8G8B8(converted, xf->width, xf->height, xf->stride, xf->data); + break; case XRT_FORMAT_BAYER_GR8:; uint32_t w = xf->width / 2; uint32_t h = xf->height / 2; @@ -567,6 +620,8 @@ receive_frame_r8g8b8(struct xrt_frame_sink *xs, struct xrt_frame *xf) return; } if (!from_MJPEG_to_R8G8B8(converted, xf->size, xf->data)) { + // Make sure to free frame when we fail to decode. + xrt_frame_reference(&converted, NULL); return; } break; @@ -581,8 +636,46 @@ receive_frame_r8g8b8(struct xrt_frame_sink *xs, struct xrt_frame *xf) } static void -receive_frame_yuv_yuyv_uyvy_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf) +convert_frame_rgb_yuv_yuyv_uyvy_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf) { + SINK_TRACE_MARKER(); + + struct u_sink_converter *s = (struct u_sink_converter *)xs; + + struct xrt_frame *converted = NULL; + + switch (xf->format) { + case XRT_FORMAT_R8G8B8: + case XRT_FORMAT_L8: + case XRT_FORMAT_YUYV422: + case XRT_FORMAT_UYVY422: + case XRT_FORMAT_YUV888: s->downstream->push_frame(s->downstream, xf); return; +#ifdef XRT_HAVE_JPEG + case XRT_FORMAT_MJPEG: + if (!create_frame_with_format(xf, XRT_FORMAT_YUV888, &converted)) { + return; + } + if (!from_MJPEG_to_YUV888(converted, xf->size, xf->data)) { + return; + } + break; +#endif + default: + U_LOG_E("Can not convert from '%s' to either R8G8B8, YUV, YUYV, UYVY or L8!", u_format_str(xf->format)); + return; + } + + s->downstream->push_frame(s->downstream, converted); + + // Refcount in case it's being held downstream. + xrt_frame_reference(&converted, NULL); +} + +static void +convert_frame_yuv_yuyv_uyvy_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *xf) +{ + SINK_TRACE_MARKER(); + struct u_sink_converter *s = (struct u_sink_converter *)xs; struct xrt_frame *converted = NULL; @@ -617,8 +710,10 @@ receive_frame_yuv_yuyv_uyvy_or_l8(struct xrt_frame_sink *xs, struct xrt_frame *x } static void -receive_frame_yuv_or_yuyv(struct xrt_frame_sink *xs, struct xrt_frame *xf) +convert_frame_yuv_or_yuyv(struct xrt_frame_sink *xs, struct xrt_frame *xf) { + SINK_TRACE_MARKER(); + struct u_sink_converter *s = (struct u_sink_converter *)xs; struct xrt_frame *converted = NULL; @@ -646,8 +741,10 @@ receive_frame_yuv_or_yuyv(struct xrt_frame_sink *xs, struct xrt_frame *xf) } XRT_MAYBE_UNUSED static void -receive_frame_bayer(struct xrt_frame_sink *xs, struct xrt_frame *xf) +convert_frame_bayer(struct xrt_frame_sink *xs, struct xrt_frame *xf) { + SINK_TRACE_MARKER(); + struct u_sink_converter *s = (struct u_sink_converter *)xs; uint32_t w = xf->width / 2; @@ -701,7 +798,7 @@ u_sink_create_format_converter(struct xrt_frame_context *xfctx, #endif struct u_sink_converter *s = U_TYPED_CALLOC(struct u_sink_converter); - s->base.push_frame = receive_frame_r8g8b8; + s->base.push_frame = convert_frame_r8g8b8; s->node.break_apart = break_apart; s->node.destroy = destroy; s->downstream = downstream; @@ -717,7 +814,7 @@ u_sink_create_to_r8g8b8_or_l8(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs) { struct u_sink_converter *s = U_TYPED_CALLOC(struct u_sink_converter); - s->base.push_frame = receive_frame_r8g8b8_or_l8; + s->base.push_frame = convert_frame_r8g8b8_or_l8; s->node.break_apart = break_apart; s->node.destroy = destroy; s->downstream = downstream; @@ -737,7 +834,7 @@ u_sink_create_to_r8g8b8_bayer_or_l8(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs) { struct u_sink_converter *s = U_TYPED_CALLOC(struct u_sink_converter); - s->base.push_frame = receive_frame_r8g8b8_bayer_or_l8; + s->base.push_frame = convert_frame_r8g8b8_bayer_or_l8; s->node.break_apart = break_apart; s->node.destroy = destroy; s->downstream = downstream; @@ -747,6 +844,21 @@ u_sink_create_to_r8g8b8_bayer_or_l8(struct xrt_frame_context *xfctx, *out_xfs = &s->base; } +void +u_sink_create_to_rgb_yuv_yuyv_uyvy_or_l8(struct xrt_frame_context *xfctx, + struct xrt_frame_sink *downstream, + struct xrt_frame_sink **out_xfs) +{ + struct u_sink_converter *s = U_TYPED_CALLOC(struct u_sink_converter); + s->base.push_frame = convert_frame_rgb_yuv_yuyv_uyvy_or_l8; + s->node.break_apart = break_apart; + s->node.destroy = destroy; + s->downstream = downstream; + + xrt_frame_context_add(xfctx, &s->node); + + *out_xfs = &s->base; +} void u_sink_create_to_yuv_yuyv_uyvy_or_l8(struct xrt_frame_context *xfctx, @@ -754,7 +866,7 @@ u_sink_create_to_yuv_yuyv_uyvy_or_l8(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs) { struct u_sink_converter *s = U_TYPED_CALLOC(struct u_sink_converter); - s->base.push_frame = receive_frame_yuv_yuyv_uyvy_or_l8; + s->base.push_frame = convert_frame_yuv_yuyv_uyvy_or_l8; s->node.break_apart = break_apart; s->node.destroy = destroy; s->downstream = downstream; @@ -770,7 +882,7 @@ u_sink_create_to_yuv_or_yuyv(struct xrt_frame_context *xfctx, struct xrt_frame_sink **out_xfs) { struct u_sink_converter *s = U_TYPED_CALLOC(struct u_sink_converter); - s->base.push_frame = receive_frame_yuv_or_yuyv; + s->base.push_frame = convert_frame_yuv_or_yuyv; s->node.break_apart = break_apart; s->node.destroy = destroy; s->downstream = downstream; diff --git a/src/xrt/auxiliary/util/u_sink_deinterleaver.c b/src/xrt/auxiliary/util/u_sink_deinterleaver.c index f00fc2dd0..cbbe1c2f6 100644 --- a/src/xrt/auxiliary/util/u_sink_deinterleaver.c +++ b/src/xrt/auxiliary/util/u_sink_deinterleaver.c @@ -1,4 +1,4 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -11,6 +11,7 @@ #include "util/u_misc.h" #include "util/u_sink.h" #include "util/u_frame.h" +#include "util/u_trace_marker.h" /*! @@ -43,6 +44,8 @@ L8_interleaved_to_L8(const uint8_t *input, uint8_t *l8a, uint8_t *l8b) static void from_L8_interleaved_to_L8(struct xrt_frame *frame, uint32_t w, uint32_t h, size_t stride, const uint8_t *data) { + SINK_TRACE_MARKER(); + uint32_t half_w = w / 2; for (uint32_t y = 0; y < h; y++) { @@ -66,8 +69,10 @@ from_L8_interleaved_to_L8(struct xrt_frame *frame, uint32_t w, uint32_t h, size_ */ static void -deinterleaves_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) +deinterleave_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) { + SINK_TRACE_MARKER(); + struct u_sink_deinterleaver *de = (struct u_sink_deinterleaver *)xfs; if (xf->stereo_format != XRT_STEREO_FORMAT_INTERLEAVED) { @@ -107,11 +112,13 @@ deinterleaves_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) } static void -break_apart(struct xrt_frame_node *node) -{} +deinterleave_break_apart(struct xrt_frame_node *node) +{ + // Noop +} static void -destroy(struct xrt_frame_node *node) +deinterleave_destroy(struct xrt_frame_node *node) { struct u_sink_deinterleaver *de = container_of(node, struct u_sink_deinterleaver, node); @@ -132,10 +139,12 @@ u_sink_deinterleaver_create(struct xrt_frame_context *xfctx, { struct u_sink_deinterleaver *de = U_TYPED_CALLOC(struct u_sink_deinterleaver); - de->base.push_frame = deinterleaves_frame; - de->node.break_apart = break_apart; - de->node.destroy = destroy; + de->base.push_frame = deinterleave_frame; + de->node.break_apart = deinterleave_break_apart; + de->node.destroy = deinterleave_destroy; de->downstream = downstream; + xrt_frame_context_add(xfctx, &de->node); + *out_xfs = &de->base; } diff --git a/src/xrt/auxiliary/util/u_sink_queue.c b/src/xrt/auxiliary/util/u_sink_queue.c index 54da53336..11151b06d 100644 --- a/src/xrt/auxiliary/util/u_sink_queue.c +++ b/src/xrt/auxiliary/util/u_sink_queue.c @@ -1,4 +1,4 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -9,22 +9,31 @@ #include "util/u_misc.h" #include "util/u_sink.h" +#include "util/u_trace_marker.h" #include #include /*! - * An @ref xrt_frame_sink queue. + * An @ref xrt_frame_sink queue, any frames received will be pushed to the + * downstream consumer on the queue thread. Will drop frames should multiple + * frames be queued up. + * * @implements xrt_frame_sink * @implements xrt_frame_node */ struct u_sink_queue { + //! Base sink. struct xrt_frame_sink base; + //! For tracking on the frame context. struct xrt_frame_node node; + //! The consumer of the frames that are queued. struct xrt_frame_sink *consumer; + + //! The current queued frame. struct xrt_frame *frame; pthread_t thread; @@ -37,12 +46,15 @@ struct u_sink_queue uint64_t last; } seq; + //! Should we keep running. bool running; }; static void * -sque_run(void *ptr) +queue_mainloop(void *ptr) { + SINK_TRACE_MARKER(); + struct u_sink_queue *q = (struct u_sink_queue *)ptr; struct xrt_frame *frame = NULL; @@ -55,7 +67,7 @@ sque_run(void *ptr) pthread_cond_wait(&q->cond, &q->mutex); } - // Where we woken up to turn of. + // Where we woken up to turn off. if (!q->running) { break; } @@ -65,24 +77,33 @@ sque_run(void *ptr) continue; } + SINK_TRACE_IDENT(queue_frame); + // We have a new frame, send it out. q->seq.last = q->seq.current; - // Take a reference on the current frame, this keeps it alive - // if it is replaced during the consumer processing it, but - // we no longer need to hold onto the frame on the queue we - // just move the pointer. + /* + * We need to take a reference on the current frame, this is to + * keep it alive during the call to the consumer should it be + * replaced. But we no longer need to hold onto the frame on the + * queue so we just move the pointer. + */ frame = q->frame; q->frame = NULL; - // Unlock the mutex when we do the work. + /* + * Unlock the mutex when we do the work, so a new frame can be + * queued. + */ pthread_mutex_unlock(&q->mutex); // Send to the consumer that does the work. q->consumer->push_frame(q->consumer, frame); - // Drop our reference we don't need it anymore, - // or it's held on the queue. + /* + * Drop our reference we don't need it anymore, or it's held by + * the consumer. + */ xrt_frame_reference(&frame, NULL); // Have to lock it again. @@ -95,8 +116,10 @@ sque_run(void *ptr) } static void -sque_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) +queue_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) { + SINK_TRACE_MARKER(); + struct u_sink_queue *q = (struct u_sink_queue *)xfs; pthread_mutex_lock(&q->mutex); @@ -114,7 +137,7 @@ sque_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) } static void -break_apart(struct xrt_frame_node *node) +queue_break_apart(struct xrt_frame_node *node) { struct u_sink_queue *q = container_of(node, struct u_sink_queue, node); void *retval = NULL; @@ -139,7 +162,7 @@ break_apart(struct xrt_frame_node *node) } static void -destroy(struct xrt_frame_node *node) +queue_destroy(struct xrt_frame_node *node) { struct u_sink_queue *q = container_of(node, struct u_sink_queue, node); @@ -162,9 +185,9 @@ u_sink_queue_create(struct xrt_frame_context *xfctx, struct xrt_frame_sink *down struct u_sink_queue *q = U_TYPED_CALLOC(struct u_sink_queue); int ret = 0; - q->base.push_frame = sque_frame; - q->node.break_apart = break_apart; - q->node.destroy = destroy; + q->base.push_frame = queue_frame; + q->node.break_apart = queue_break_apart; + q->node.destroy = queue_destroy; q->consumer = downstream; q->running = true; @@ -181,7 +204,7 @@ u_sink_queue_create(struct xrt_frame_context *xfctx, struct xrt_frame_sink *down return false; } - ret = pthread_create(&q->thread, NULL, sque_run, q); + ret = pthread_create(&q->thread, NULL, queue_mainloop, q); if (ret != 0) { pthread_cond_destroy(&q->cond); pthread_mutex_destroy(&q->mutex); diff --git a/src/xrt/auxiliary/util/u_sink_quirk.c b/src/xrt/auxiliary/util/u_sink_quirk.c index 2644f1f99..095d0e482 100644 --- a/src/xrt/auxiliary/util/u_sink_quirk.c +++ b/src/xrt/auxiliary/util/u_sink_quirk.c @@ -109,5 +109,7 @@ u_sink_quirk_create(struct xrt_frame_context *xfctx, q->ps4_cam = params->ps4_cam; q->leap_motion = params->leap_motion; + xrt_frame_context_add(xfctx, &q->node); + *out_xfs = &q->base; } diff --git a/src/xrt/auxiliary/util/u_sink_split.c b/src/xrt/auxiliary/util/u_sink_split.c index fa8b00506..a37878ff5 100644 --- a/src/xrt/auxiliary/util/u_sink_split.c +++ b/src/xrt/auxiliary/util/u_sink_split.c @@ -1,4 +1,4 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -9,6 +9,7 @@ #include "util/u_misc.h" #include "util/u_sink.h" +#include "util/u_trace_marker.h" /*! @@ -28,6 +29,8 @@ struct u_sink_split static void split_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) { + SINK_TRACE_MARKER(); + struct u_sink_split *s = (struct u_sink_split *)xfs; s->left->push_frame(s->left, xf); @@ -35,11 +38,13 @@ split_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) } static void -break_apart(struct xrt_frame_node *node) -{} +split_break_apart(struct xrt_frame_node *node) +{ + // Noop +} static void -destroy(struct xrt_frame_node *node) +split_destroy(struct xrt_frame_node *node) { struct u_sink_split *s = container_of(node, struct u_sink_split, node); @@ -62,10 +67,12 @@ u_sink_split_create(struct xrt_frame_context *xfctx, struct u_sink_split *s = U_TYPED_CALLOC(struct u_sink_split); s->base.push_frame = split_frame; - s->node.break_apart = break_apart; - s->node.destroy = destroy; + s->node.break_apart = split_break_apart; + s->node.destroy = split_destroy; s->left = left; s->right = right; + xrt_frame_context_add(xfctx, &s->node); + *out_xfs = &s->base; } diff --git a/src/xrt/auxiliary/util/u_template_historybuf.hpp b/src/xrt/auxiliary/util/u_template_historybuf.hpp new file mode 100644 index 000000000..bafe304dd --- /dev/null +++ b/src/xrt/auxiliary/util/u_template_historybuf.hpp @@ -0,0 +1,73 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Ringbuffer implementation for keeping track of the past state of things + * @author Moses Turner + * @ingroup drv_ht + */ + +#pragma once + +#include +#include + + +//| -4 | -3 | -2 | -1 | Top | Garbage | +// OR +//| -4 | -3 | -2 | -1 | Top | -7 | -6 | -5 | + +namespace xrt::auxiliary::util { + +template class HistoryBuffer +{ +private: + T internalBuffer[maxSize]; + int mTopIdx = 0; + int mLength = 0; + + +public: + // clang-format off + int topIdx() { return mTopIdx; } + int length() { return mLength; } + // clang-format on + + /* Put something at the top, overwrite whatever was at the back*/ + void + push(const T inElement) + { + mTopIdx++; + if (mTopIdx == maxSize) { + mTopIdx = 0; + } + + memcpy(&internalBuffer[mTopIdx], &inElement, sizeof(T)); + mLength++; + mLength = std::min(mLength, maxSize); + } + + T * // Hack comment to fix clang-format + operator[](int inIndex) + { + if (mLength == 0) { + return NULL; + } + assert(inIndex <= maxSize); + assert(inIndex >= 0); + + int index = mTopIdx - inIndex; + if (index < 0) { + index = maxSize + index; + } + + assert(index >= 0); + if (index > maxSize) { + assert(false); + } + + return &internalBuffer[index]; + } +}; + +} // namespace xrt::auxiliary::util diff --git a/src/xrt/auxiliary/util/u_time.h b/src/xrt/auxiliary/util/u_time.h index 9a2ccb156..e5abb5ac9 100644 --- a/src/xrt/auxiliary/util/u_time.h +++ b/src/xrt/auxiliary/util/u_time.h @@ -24,13 +24,30 @@ extern "C" { #endif +//! Helper define to make code more readable. +#define U_1_000_000_000 (1000000000) + /*! * The number of nanoseconds in a second. * * @see timepoint_ns * @ingroup time_ns_to_s */ -#define U_1_000_000_000 (1000000000) +#define U_TIME_1S_IN_NS U_1_000_000_000 + +/*! + * The number of nanoseconds in a milliseconds. + * + * @see timepoint_ns + */ +#define U_TIME_1MS_IN_NS (U_TIME_1S_IN_NS / 1000) + +/*! + * The number of nanoseconds in half a milliseconds. + * + * @see timepoint_ns + */ +#define U_TIME_HALF_MS_IN_NS (U_TIME_1MS_IN_NS / 2) /*! * Integer timestamp type. @@ -53,15 +70,15 @@ typedef int64_t timepoint_ns; typedef int64_t time_duration_ns; /*! - * Convert nanoseconds duration to float seconds. + * Convert nanoseconds duration to double seconds. * * @see timepoint_ns * @ingroup aux_util */ -static inline float +static inline double time_ns_to_s(time_duration_ns ns) { - return (float)(ns) / (float)U_1_000_000_000; + return (double)(ns) / (double)U_TIME_1S_IN_NS; } /*! @@ -73,7 +90,92 @@ time_ns_to_s(time_duration_ns ns) static inline time_duration_ns time_s_to_ns(double duration) { - return (time_duration_ns)(duration * (double)U_1_000_000_000); + return (time_duration_ns)(duration * (double)U_TIME_1S_IN_NS); +} + +/*! + * Convert nanoseconds to double float milliseconds, useful for printing. + * + * @see timepoint_ns + * @ingroup aux_util + */ +static inline double +time_ns_to_ms_f(time_duration_ns ns) +{ + return (double)(ns) / (double)U_TIME_1MS_IN_NS; +} + +/*! + * Checks if two timepoints are with a certain range of each other. + * + * @see timepoint_ns + * @ingroup aux_util + */ +static inline bool +time_is_within_range_of_each_other(timepoint_ns a, timepoint_ns b, uint64_t range) +{ + int64_t t = (int64_t)a - (int64_t)b; + return (-(int64_t)range < t) && (t < (int64_t)range); +} + +/*! + * Checks if two timepoints are with half a millisecond of each other. + * + * @see timepoint_ns + * @ingroup aux_util + */ +static inline bool +time_is_within_half_ms(timepoint_ns a, timepoint_ns b) +{ + return time_is_within_range_of_each_other(a, b, U_TIME_HALF_MS_IN_NS); +} + +/*! + * Fuzzy comparisons. + * + * @see timepoint_ns + * @ingroup aux_util + */ +static inline bool +time_is_less_then_or_within_range(timepoint_ns a, timepoint_ns b, uint64_t range) +{ + return a < b || time_is_within_range_of_each_other(a, b, range); +} + +/*! + * Fuzzy comparisons. + * + * @see timepoint_ns + * @ingroup aux_util + */ +static inline bool +time_is_less_then_or_within_half_ms(timepoint_ns a, timepoint_ns b) +{ + return time_is_less_then_or_within_range(a, b, U_TIME_HALF_MS_IN_NS); +} + +/*! + * Fuzzy comparisons. + * + * @see timepoint_ns + * @ingroup aux_util + */ +static inline bool +time_is_greater_then_or_within_range(timepoint_ns a, timepoint_ns b, uint64_t range) +{ + return a > b || time_is_within_range_of_each_other(a, b, range); +} + +/*! + * Fuzzy comparisons. + * + * @see timepoint_ns + * @ingroup aux_util + */ +static inline bool +time_is_greater_then_or_within_half_ms(timepoint_ns a, timepoint_ns b) +{ + return time_is_greater_then_or_within_range(a, b, U_TIME_HALF_MS_IN_NS); } /*! diff --git a/src/xrt/auxiliary/util/u_timing.h b/src/xrt/auxiliary/util/u_timing.h new file mode 100644 index 000000000..d1ee0f5c8 --- /dev/null +++ b/src/xrt/auxiliary/util/u_timing.h @@ -0,0 +1,404 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Shared timing code. + * @author Jakob Bornecrantz + * @ingroup aux_util + */ + +#pragma once + +#include "xrt/xrt_compiler.h" +#include "xrt/xrt_defines.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +/*! + * @defgroup aux_timing Frame and Render timing + * + * @ingroup aux_util + * @see @ref frame-timing. + */ + + +/*! + * For marking timepoints on a frame's lifetime, not a async event. + * + * @ingroup aux_timing + */ +enum u_timing_point +{ + U_TIMING_POINT_WAKE_UP, //!< Woke up after sleeping in wait frame. + U_TIMING_POINT_BEGIN, //!< Began CPU side work for GPU. + U_TIMING_POINT_SUBMIT, //!< Submitted work to the GPU. +}; + + +/* + * + * Frame timing helper. + * + */ + +/*! + * Frame timing helper struct, used for the compositors own frame timing. + * + * @ingroup aux_timing + */ +struct u_frame_timing +{ + /*! + * Predict the next frame. + * + * @param[in] uft The frame timing struct. + * @param[out] out_frame_id Id used to refer to this frame again. + * @param[out] out_wake_up_time_ns When should the compositor wake up. + * @param[out] out_desired_present_time_ns The GPU should start scanning out at this time. + * @param[out] out_present_slop_ns Any looseness to the desired present timing. + * @param[out] out_predicted_display_time_ns At what time have we predicted that pixels turns to photons. + * @param[out] out_predicted_display_period_ns Display period that we are running on. + * @param[out] out_min_display_period_ns The fastest theoretical display period. + * + * @see @ref frame-timing. + */ + void (*predict)(struct u_frame_timing *uft, + int64_t *out_frame_id, + uint64_t *out_wake_up_time_ns, + uint64_t *out_desired_present_time_ns, + uint64_t *out_present_slop_ns, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns, + uint64_t *out_min_display_period_ns); + + /*! + * Mark a point on the frame's lifetime. + * + * @see @ref frame-timing. + */ + void (*mark_point)(struct u_frame_timing *uft, enum u_timing_point point, int64_t frame_id, uint64_t when_ns); + + /*! + * Provide frame timing information about a delivered frame, this is + * usually provided by the display system. These arguments currently + * matches 1-to-1 what VK_GOOGLE_display_timing provides. + * + * Depend on when the information is delivered this can be called at any + * point of the following frames. + * + * @see @ref frame-timing. + */ + void (*info)(struct u_frame_timing *uft, + int64_t frame_id, + uint64_t desired_present_time_ns, + uint64_t actual_present_time_ns, + uint64_t earliest_present_time_ns, + uint64_t present_margin_ns); + + /*! + * Destroy this u_frame_timing. + */ + void (*destroy)(struct u_frame_timing *uft); +}; + +/*! + * @copydoc u_frame_timing::predict + * + * Helper for calling through the function pointer. + * + * @public @memberof u_frame_timing + * @ingroup aux_timing + */ +static inline void +u_ft_predict(struct u_frame_timing *uft, + int64_t *out_frame_id, + uint64_t *out_wake_up_time_ns, + uint64_t *out_desired_present_time_ns, + uint64_t *out_present_slop_ns, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns, + uint64_t *out_min_display_period_ns) +{ + uft->predict(uft, // + out_frame_id, // + out_wake_up_time_ns, // + out_desired_present_time_ns, // + out_present_slop_ns, // + out_predicted_display_time_ns, // + out_predicted_display_period_ns, // + out_min_display_period_ns); // +} + +/*! + * @copydoc u_frame_timing::mark_point + * + * Helper for calling through the function pointer. + * + * @public @memberof u_frame_timing + * @ingroup aux_timing + */ +static inline void +u_ft_mark_point(struct u_frame_timing *uft, enum u_timing_point point, int64_t frame_id, uint64_t when_ns) +{ + uft->mark_point(uft, point, frame_id, when_ns); +} + +/*! + * @copydoc u_frame_timing::info + * + * Helper for calling through the function pointer. + * + * @public @memberof u_frame_timing + * @ingroup aux_timing + */ +static inline void +u_ft_info(struct u_frame_timing *uft, + int64_t frame_id, + uint64_t desired_present_time_ns, + uint64_t actual_present_time_ns, + uint64_t earliest_present_time_ns, + uint64_t present_margin_ns) +{ + uft->info(uft, frame_id, desired_present_time_ns, actual_present_time_ns, earliest_present_time_ns, + present_margin_ns); +} + +/*! + * @copydoc u_frame_timing::destroy + * + * Helper for calling through the function pointer: does a null check and sets + * uft_ptr to null if freed. + * + * @public @memberof u_frame_timing + * @ingroup aux_timing + */ +static inline void +u_ft_destroy(struct u_frame_timing **uft_ptr) +{ + struct u_frame_timing *uft = *uft_ptr; + if (uft == NULL) { + return; + } + + uft->destroy(uft); + *uft_ptr = NULL; +} + + +/* + * + * Render timing helper. + * + */ + +/*! + * This render timing helper is designed to schedule the rendering time of + * clients that submit frames to a compositor, which runs its own render loop + * that picks latest completed frames for that client. + * + * @ingroup aux_timing + */ +struct u_render_timing +{ + /*! + * Predict when the client's next: rendered frame will be display; when the + * client should be woken up from sleeping; and its display period. + * + * This is called from `xrWaitFrame`, but it does not do any waiting, the caller + * should wait till `out_wake_up_time`. + * + * @param urt Render timing helper. + * @param[out] out_frame_id Frame ID of this predicted frame. + * @param[out] out_wake_up_time When the client should be woken up. + * @param[out] out_predicted_display_time Predicted display time. + * @param[out] out_predicted_display_period Predicted display period. + */ + void (*predict)(struct u_render_timing *urt, + int64_t *out_frame_id, + uint64_t *out_wake_up_time, + uint64_t *out_predicted_display_time, + uint64_t *out_predicted_display_period); + + /*! + * Mark a point on the frame's lifetime. + * + * @see @ref frame-timing. + */ + void (*mark_point)(struct u_render_timing *urt, int64_t frame_id, enum u_timing_point point, uint64_t when_ns); + + /*! + * When a frame has been discarded. + */ + void (*mark_discarded)(struct u_render_timing *urt, int64_t frame_id); + + /*! + * A frame has been delivered from the client, see `xrEndFrame`. The GPU might + * still be rendering the work. + */ + void (*mark_delivered)(struct u_render_timing *urt, int64_t frame_id); + + /*! + * Add a new sample point from the main render loop. + * + * This is called in the main renderer loop that tightly submits frames to the + * real compositor for displaying. This is only used to inform the render helper + * when the frame will be shown, not any timing information about the client. + * + * When this is called doesn't matter that much, as the render timing will need + * to be able to predict one or more frames into the future anyways. But + * preferably as soon as the main loop wakes up from wait frame. + * + * @param urt Self pointer + * @param predicted_display_time_ns Predicted display time for this sample. + * @param predicted_display_period_ns Predicted display period for this sample. + * @param extra_ns Time between display and when this sample + * was created, that is when the main loop + * was woken up by the main compositor. + */ + void (*info)(struct u_render_timing *urt, + uint64_t predicted_display_time_ns, + uint64_t predicted_display_period_ns, + uint64_t extra_ns); + + /*! + * Destroy this u_render_timing. + */ + void (*destroy)(struct u_render_timing *urt); +}; + +/*! + * @copydoc u_render_timing::predict + * + * Helper for calling through the function pointer. + * + * @public @memberof u_render_timing + * @ingroup aux_timing + */ +static inline void +u_rt_predict(struct u_render_timing *urt, + int64_t *out_frame_id, + uint64_t *out_wake_up_time, + uint64_t *out_predicted_display_time, + uint64_t *out_predicted_display_period) +{ + urt->predict(urt, out_frame_id, out_wake_up_time, out_predicted_display_time, out_predicted_display_period); +} + +/*! + * @copydoc u_render_timing::mark_point + * + * Helper for calling through the function pointer. + * + * @public @memberof u_render_timing + * @ingroup aux_timing + */ +static inline void +u_rt_mark_point(struct u_render_timing *urt, int64_t frame_id, enum u_timing_point point, uint64_t when_ns) +{ + urt->mark_point(urt, frame_id, point, when_ns); +} + +/*! + * @copydoc u_render_timing::mark_discarded + * + * Helper for calling through the function pointer. + * + * @public @memberof u_render_timing + * @ingroup aux_timing + */ +static inline void +u_rt_mark_discarded(struct u_render_timing *urt, int64_t frame_id) +{ + urt->mark_discarded(urt, frame_id); +} + +/*! + * @copydoc u_render_timing::mark_delivered + * + * Helper for calling through the function pointer. + * + * @public @memberof u_render_timing + * @ingroup aux_timing + */ +static inline void +u_rt_mark_delivered(struct u_render_timing *urt, int64_t frame_id) +{ + urt->mark_delivered(urt, frame_id); +} + +/*! + * @copydoc u_render_timing::info + * + * Helper for calling through the function pointer. + * + * @public @memberof u_render_timing + * @ingroup aux_timing + */ +static inline void +u_rt_info(struct u_render_timing *urt, + uint64_t predicted_display_time_ns, + uint64_t predicted_display_period_ns, + uint64_t extra_ns) +{ + urt->info(urt, predicted_display_time_ns, predicted_display_period_ns, extra_ns); +} + +/*! + * @copydoc u_render_timing::destroy + * + * Helper for calling through the function pointer: does a null check and sets + * urt_ptr to null if freed. + * + * @public @memberof u_frame_timing + * @ingroup aux_timing + */ +static inline void +u_rt_destroy(struct u_render_timing **urt_ptr) +{ + struct u_render_timing *urt = *urt_ptr; + if (urt == NULL) { + return; + } + + urt->destroy(urt); + *urt_ptr = NULL; +} + + +/* + * + * Implementations. + * + */ + +/*! + * Meant to be used with VK_GOOGLE_display_timing. + * + * @ingroup aux_timing + */ +xrt_result_t +u_ft_display_timing_create(uint64_t estimated_frame_period_ns, struct u_frame_timing **out_uft); + +/*! + * When you can not get display timing information use this. + * + * @ingroup aux_timing + */ +xrt_result_t +u_ft_fake_create(uint64_t estimated_frame_period_ns, struct u_frame_timing **out_uft); + +/*! + * Creates a new render timing. + * + * @ingroup aux_timing + */ +xrt_result_t +u_rt_create(struct u_render_timing **out_urt); + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/util/u_timing_fake.c b/src/xrt/auxiliary/util/u_timing_fake.c new file mode 100644 index 000000000..211bcf511 --- /dev/null +++ b/src/xrt/auxiliary/util/u_timing_fake.c @@ -0,0 +1,196 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief For generating a fake timing. + * @author Jakob Bornecrantz + * @ingroup aux_util + */ + +#include "os/os_time.h" + +#include "util/u_time.h" +#include "util/u_misc.h" +#include "util/u_debug.h" +#include "util/u_timing.h" +#include "util/u_logging.h" +#include "util/u_trace_marker.h" + +#include +#include +#include + + +/* + * + * Structs and defines. + * + */ + + +struct fake_timing +{ + struct u_frame_timing base; + + /*! + * The periodicity of the display. + */ + uint64_t frame_period_ns; + + /*! + * When the last frame was made. + */ + uint64_t last_display_time_ns; + + /*! + * Very often the present time that we get from the system is only when + * the display engine starts scanning out from the buffers we provided, + * and not when the pixels turned into photons that the user sees. + */ + uint64_t present_offset_ns; + + // The amount of time that the application needs to render frame. + uint64_t app_time_ns; + + int64_t frame_id_generator; +}; + + +/* + * + * Helper functions. + * + */ + +static inline struct fake_timing * +fake_timing(struct u_frame_timing *uft) +{ + return (struct fake_timing *)uft; +} + +static uint64_t +predict_next_frame(struct fake_timing *ft) +{ + uint64_t time_needed_ns = ft->present_offset_ns + ft->app_time_ns; + uint64_t now_ns = os_monotonic_get_ns(); + uint64_t predicted_display_time_ns = ft->last_display_time_ns + ft->frame_period_ns; + + while (now_ns + time_needed_ns > predicted_display_time_ns) { + predicted_display_time_ns += ft->frame_period_ns; + } + + return predicted_display_time_ns; +} + +static uint64_t +get_percent_of_time(uint64_t time_ns, uint32_t fraction_percent) +{ + double fraction = (double)fraction_percent / 100.0; + return time_s_to_ns(time_ns_to_s(time_ns) * fraction); +} + + +/* + * + * Member functions. + * + */ + +static void +ft_predict(struct u_frame_timing *uft, + int64_t *out_frame_id, + uint64_t *out_wake_up_time_ns, + uint64_t *out_desired_present_time_ns, + uint64_t *out_present_slop_ns, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns, + uint64_t *out_min_display_period_ns) +{ + struct fake_timing *ft = fake_timing(uft); + + int64_t frame_id = ft->frame_id_generator++; + uint64_t predicted_display_time_ns = predict_next_frame(ft); + uint64_t desired_present_time_ns = predicted_display_time_ns - ft->present_offset_ns; + uint64_t wake_up_time_ns = desired_present_time_ns - ft->app_time_ns; + uint64_t present_slop_ns = U_TIME_HALF_MS_IN_NS; + uint64_t predicted_display_period_ns = ft->frame_period_ns; + uint64_t min_display_period_ns = ft->frame_period_ns; + + *out_frame_id = frame_id; + *out_wake_up_time_ns = wake_up_time_ns; + *out_desired_present_time_ns = desired_present_time_ns; + *out_present_slop_ns = present_slop_ns; + *out_predicted_display_time_ns = predicted_display_time_ns; + *out_predicted_display_period_ns = predicted_display_period_ns; + *out_min_display_period_ns = min_display_period_ns; +} + +static void +ft_mark_point(struct u_frame_timing *uft, enum u_timing_point point, int64_t frame_id, uint64_t when_ns) +{ + // To help validate calling code. + switch (point) { + case U_TIMING_POINT_WAKE_UP: break; + case U_TIMING_POINT_BEGIN: break; + case U_TIMING_POINT_SUBMIT: break; + default: assert(false); + } +} + +static void +ft_info(struct u_frame_timing *uft, + int64_t frame_id, + uint64_t desired_present_time_ns, + uint64_t actual_present_time_ns, + uint64_t earliest_present_time_ns, + uint64_t present_margin_ns) +{ + /* + * The compositor might call this function because it selected the + * fake timing code even tho displaying timing is available. + */ +} + +static void +ft_destroy(struct u_frame_timing *uft) +{ + struct fake_timing *ft = fake_timing(uft); + free(ft); +} + + +/* + * + * 'Exported' functions. + * + */ + +xrt_result_t +u_ft_fake_create(uint64_t estimated_frame_period_ns, struct u_frame_timing **out_uft) +{ + struct fake_timing *ft = U_TYPED_CALLOC(struct fake_timing); + ft->base.predict = ft_predict; + ft->base.mark_point = ft_mark_point; + ft->base.info = ft_info; + ft->base.destroy = ft_destroy; + ft->frame_period_ns = estimated_frame_period_ns; + + // To make sure the code can start from a non-zero frame id. + ft->frame_id_generator = 5; + + // Just a wild guess. + ft->present_offset_ns = U_TIME_1MS_IN_NS * 4; + + // 20% of the frame time. + ft->app_time_ns = get_percent_of_time(estimated_frame_period_ns, 20); + + // Make the next display time be in the future. + ft->last_display_time_ns = os_monotonic_get_ns() + U_TIME_1MS_IN_NS * 50.0; + + // Return value. + *out_uft = &ft->base; + + U_LOG_I("Created fake timing"); + + return XRT_SUCCESS; +} diff --git a/src/xrt/auxiliary/util/u_timing_frame.c b/src/xrt/auxiliary/util/u_timing_frame.c new file mode 100644 index 000000000..506b79728 --- /dev/null +++ b/src/xrt/auxiliary/util/u_timing_frame.c @@ -0,0 +1,622 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Shared frame timing code. + * @author Jakob Bornecrantz + * @ingroup aux_util + */ + +#include "os/os_time.h" + +#include "util/u_time.h" +#include "util/u_misc.h" +#include "util/u_debug.h" +#include "util/u_timing.h" +#include "util/u_logging.h" +#include "util/u_trace_marker.h" + +#include +#include +#include + +DEBUG_GET_ONCE_LOG_OPTION(ll, "U_TIMING_FRAME_LOG", U_LOGGING_WARN) + +#define FT_LOG_T(...) U_LOG_IFL_T(debug_get_log_option_ll(), __VA_ARGS__) +#define FT_LOG_D(...) U_LOG_IFL_D(debug_get_log_option_ll(), __VA_ARGS__) +#define FT_LOG_I(...) U_LOG_IFL_I(debug_get_log_option_ll(), __VA_ARGS__) +#define FT_LOG_W(...) U_LOG_IFL_W(debug_get_log_option_ll(), __VA_ARGS__) +#define FT_LOG_E(...) U_LOG_IFL_E(debug_get_log_option_ll(), __VA_ARGS__) + +#define NUM_FRAMES 16 + + +/* + * + * Display timing code. + * + */ + +enum frame_state +{ + STATE_SKIPPED = -1, + STATE_CLEARED = 0, + STATE_PREDICTED = 1, + STATE_WOKE = 2, + STATE_BEGAN = 3, + STATE_SUBMITTED = 4, + STATE_INFO = 5, +}; + +struct frame +{ + int64_t frame_id; + uint64_t when_predict_ns; + uint64_t wake_up_time_ns; + uint64_t when_woke_ns; + uint64_t when_began_ns; + uint64_t when_submitted_ns; + uint64_t when_infoed_ns; + uint64_t current_app_time_ns; + uint64_t expected_done_time_ns; //!< When we expect the compositor to be done with it's frame. + uint64_t desired_present_time_ns; + uint64_t predicted_display_time_ns; + uint64_t present_margin_ns; + uint64_t actual_present_time_ns; + uint64_t earliest_present_time_ns; + + enum frame_state state; +}; + +struct display_timing +{ + struct u_frame_timing base; + + /*! + * Very often the present time that we get from the system is only when + * the display engine starts scanning out from the buffers we provided, + * and not when the pixels turned into photons that the user sees. + */ + uint64_t present_offset_ns; + + /*! + * Frame period of the device. + */ + uint64_t frame_period_ns; + + /*! + * The amount of time that the application needs to render frame. + */ + uint64_t app_time_ns; + + /*! + * The amount of time that the application needs to render frame. + */ + uint64_t padding_time_ns; + + /*! + * Used to generate frame IDs. + */ + int64_t next_frame_id; + + /*! + * The maximum amount we give to the 'app'. + */ + uint64_t app_time_max_ns; + + /*! + * If we missed a frame, back off this much. + */ + uint64_t adjust_missed_ns; + + /*! + * Adjustment of time if we didn't miss the frame, + * also used as range to stay around timing target. + */ + uint64_t adjust_non_miss_ns; + + /*! + * Exrta time between end of draw time and when the present happens. + */ + uint64_t margin_ns; + + /*! + * Frame store. + */ + struct frame frames[NUM_FRAMES]; +}; + + +/* + * + * Helper functions. + * + */ + +static inline struct display_timing * +display_timing(struct u_frame_timing *uft) +{ + return (struct display_timing *)uft; +} + +static double +ns_to_ms(int64_t t) +{ + return (double)(t / 1000) / 1000.0; +} + +static uint64_t +get_percent_of_time(uint64_t time_ns, uint32_t fraction_percent) +{ + double fraction = (double)fraction_percent / 100.0; + return time_s_to_ns(time_ns_to_s(time_ns) * fraction); +} + +static uint64_t +calc_total_app_time(struct display_timing *dt) +{ + return dt->app_time_ns + dt->margin_ns; +} + +static uint64_t +calc_display_time_from_present_time(struct display_timing *dt, uint64_t desired_present_time_ns) +{ + return desired_present_time_ns + dt->present_offset_ns; +} + +static inline bool +is_within_of_each_other(uint64_t l, uint64_t r, uint64_t range) +{ + int64_t t = (int64_t)l - (int64_t)r; + return (-(int64_t)range < t) && (t < (int64_t)range); +} + +static inline bool +is_within_half_ms(uint64_t l, uint64_t r) +{ + return is_within_of_each_other(l, r, U_TIME_HALF_MS_IN_NS); +} + +static struct frame * +get_frame(struct display_timing *dt, int64_t frame_id) +{ + assert(frame_id >= 0); + assert((uint64_t)frame_id <= (uint64_t)SIZE_MAX); + + size_t index = (size_t)(frame_id % NUM_FRAMES); + + return &dt->frames[index]; +} + +static struct frame * +create_frame(struct display_timing *dt, enum frame_state state) +{ + int64_t frame_id = dt->next_frame_id++; + struct frame *f = get_frame(dt, frame_id); + + f->frame_id = frame_id; + f->state = state; + + return f; +} + +static struct frame * +get_latest_frame_with_state_at_least(struct display_timing *dt, enum frame_state state) +{ + uint64_t start_from = dt->next_frame_id; + uint64_t count = 1; + + while (start_from >= count && count < NUM_FRAMES) { + struct frame *f = get_frame(dt, start_from - count++); + if (f->state >= state) { + return f; + } + } + + return NULL; +} + +static struct frame * +do_clean_slate_frame(struct display_timing *dt) +{ + struct frame *f = create_frame(dt, STATE_PREDICTED); + uint64_t now_ns = os_monotonic_get_ns(); + + // Wild shot in the dark. + uint64_t the_time_ns = os_monotonic_get_ns() + dt->frame_period_ns * 10; + f->when_predict_ns = now_ns; + f->desired_present_time_ns = the_time_ns; + + return f; +} + +static struct frame * +walk_forward_through_frames(struct display_timing *dt, uint64_t last_present_time_ns) +{ + uint64_t now_ns = os_monotonic_get_ns(); + uint64_t from_time_ns = now_ns + calc_total_app_time(dt); + uint64_t desired_present_time_ns = last_present_time_ns + dt->frame_period_ns; + + while (desired_present_time_ns <= from_time_ns) { + FT_LOG_D( + "Skipped!" // + "\n\tfrom_time_ns: %" PRIu64 // + "\n\tdesired_present_time_ns: %" PRIu64 // + "\n\tdiff_ms: %.2f", // + from_time_ns, // + desired_present_time_ns, // + ns_to_ms(from_time_ns - desired_present_time_ns)); // + + // Try next frame period. + desired_present_time_ns += dt->frame_period_ns; + } + + struct frame *f = create_frame(dt, STATE_PREDICTED); + f->when_predict_ns = now_ns; + f->desired_present_time_ns = desired_present_time_ns; + + return f; +} + +static struct frame * +predict_next_frame(struct display_timing *dt) +{ + struct frame *f = NULL; + // Last earliest display time, can be zero. + struct frame *last_predicted = get_latest_frame_with_state_at_least(dt, STATE_PREDICTED); + struct frame *last_completed = get_latest_frame_with_state_at_least(dt, STATE_INFO); + if (last_predicted == NULL && last_completed == NULL) { + f = do_clean_slate_frame(dt); + } else if (last_completed == last_predicted) { + // Very high propability that we missed a frame. + f = walk_forward_through_frames(dt, last_completed->earliest_present_time_ns); + } else if (last_completed != NULL) { + assert(last_predicted != NULL); + assert(last_predicted->frame_id > last_completed->frame_id); + + int64_t diff_id = last_predicted->frame_id - last_completed->frame_id; + int64_t diff_ns = last_completed->desired_present_time_ns - last_completed->earliest_present_time_ns; + uint64_t adjusted_last_present_time_ns = + last_completed->earliest_present_time_ns + diff_id * dt->frame_period_ns; + + if (diff_ns > U_TIME_1MS_IN_NS) { + FT_LOG_D("Large diff!"); + } + if (diff_id > 1) { + FT_LOG_D( + "diff_id > 1\n" + "\tdiff_id: %" PRIi64 + "\n" + "\tadjusted_last_present_time_ns: %" PRIu64, + diff_id, adjusted_last_present_time_ns); + } + + if (diff_id > 1) { + diff_id = 1; + } + + f = walk_forward_through_frames(dt, adjusted_last_present_time_ns); + } else { + assert(last_predicted != NULL); + + f = walk_forward_through_frames(dt, last_predicted->predicted_display_time_ns); + } + + f->predicted_display_time_ns = calc_display_time_from_present_time(dt, f->desired_present_time_ns); + f->wake_up_time_ns = f->desired_present_time_ns - calc_total_app_time(dt); + f->current_app_time_ns = dt->app_time_ns; + + return f; +} + +static void +adjust_app_time(struct display_timing *dt, struct frame *f) +{ + uint64_t app_time_ns = dt->app_time_ns; + + if (f->actual_present_time_ns > f->desired_present_time_ns && + !is_within_half_ms(f->actual_present_time_ns, f->desired_present_time_ns)) { + double missed_ms = ns_to_ms(f->actual_present_time_ns - f->desired_present_time_ns); + FT_LOG_W("Frame %" PRIu64 " missed by %.2f!", f->frame_id, missed_ms); + + app_time_ns += dt->adjust_missed_ns; + if (app_time_ns > dt->app_time_max_ns) { + app_time_ns = dt->app_time_max_ns; + } + + dt->app_time_ns = app_time_ns; + return; + } + + // We want the GPU work to stop at margin_ns. + if (is_within_of_each_other( // + f->present_margin_ns, // + dt->margin_ns, // + dt->adjust_non_miss_ns)) { + // Nothing to do, the GPU ended it's work +-adjust_non_miss_ns + // of margin_ns before the present started. + return; + } + + // We didn't miss the frame but we were outside the range adjust the app time. + if (f->present_margin_ns > dt->margin_ns) { + // Approach the present time. + dt->app_time_ns -= dt->adjust_non_miss_ns; + } else { + // Back off the present time. + dt->app_time_ns += dt->adjust_non_miss_ns; + } +} + + +/* + * + * Member functions. + * + */ + +static void +dt_predict(struct u_frame_timing *uft, + int64_t *out_frame_id, + uint64_t *out_wake_up_time_ns, + uint64_t *out_desired_present_time_ns, + uint64_t *out_present_slop_ns, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns, + uint64_t *out_min_display_period_ns) +{ + struct display_timing *dt = display_timing(uft); + + struct frame *f = predict_next_frame(dt); + + uint64_t wake_up_time_ns = f->wake_up_time_ns; + uint64_t desired_present_time_ns = f->desired_present_time_ns; + uint64_t present_slop_ns = U_TIME_HALF_MS_IN_NS; + uint64_t predicted_display_time_ns = f->predicted_display_time_ns; + uint64_t predicted_display_period_ns = dt->frame_period_ns; + uint64_t min_display_period_ns = dt->frame_period_ns; + + *out_frame_id = f->frame_id; + *out_wake_up_time_ns = wake_up_time_ns; + *out_desired_present_time_ns = desired_present_time_ns; + *out_present_slop_ns = present_slop_ns; + *out_predicted_display_time_ns = predicted_display_time_ns; + *out_predicted_display_period_ns = predicted_display_period_ns; + *out_min_display_period_ns = min_display_period_ns; +} + +static void +dt_mark_point(struct u_frame_timing *uft, enum u_timing_point point, int64_t frame_id, uint64_t when_ns) +{ + struct display_timing *dt = display_timing(uft); + struct frame *f = get_frame(dt, frame_id); + + switch (point) { + case U_TIMING_POINT_WAKE_UP: + assert(f->state == STATE_PREDICTED); + f->state = STATE_WOKE; + f->when_woke_ns = when_ns; + break; + case U_TIMING_POINT_BEGIN: + assert(f->state == STATE_WOKE); + f->state = STATE_BEGAN; + f->when_began_ns = when_ns; + break; + case U_TIMING_POINT_SUBMIT: + assert(f->state == STATE_BEGAN); + f->state = STATE_SUBMITTED; + f->when_submitted_ns = when_ns; + break; + default: assert(false); + } +} + +static void +dt_info(struct u_frame_timing *uft, + int64_t frame_id, + uint64_t desired_present_time_ns, + uint64_t actual_present_time_ns, + uint64_t earliest_present_time_ns, + uint64_t present_margin_ns) +{ + struct display_timing *dt = display_timing(uft); + (void)dt; + + struct frame *last = get_latest_frame_with_state_at_least(dt, STATE_INFO); + struct frame *f = get_frame(dt, frame_id); + assert(f->state == STATE_SUBMITTED); + + f->when_infoed_ns = os_monotonic_get_ns(); + f->actual_present_time_ns = actual_present_time_ns; + f->earliest_present_time_ns = earliest_present_time_ns; + f->present_margin_ns = present_margin_ns; + f->state = STATE_INFO; + + uint64_t since_last_frame_ns = 0; + if (last != NULL) { + since_last_frame_ns = f->desired_present_time_ns - last->desired_present_time_ns; + } + + // Adjust the frame timing. + adjust_app_time(dt, f); + + double present_margin_ms = ns_to_ms(present_margin_ns); + double since_last_frame_ms = ns_to_ms(since_last_frame_ns); + + FT_LOG_T( + "Got" + "\n\tframe_id: 0x%08" PRIx64 // + "\n\twhen_predict_ns: %" PRIu64 // + "\n\twhen_woke_ns: %" PRIu64 // + "\n\twhen_submitted_ns: %" PRIu64 // + "\n\twhen_infoed_ns: %" PRIu64 // + "\n\tsince_last_frame_ms: %.2fms" // + "\n\tdesired_present_time_ns: %" PRIu64 // + "\n\tactual_present_time_ns: %" PRIu64 // + "\n\tearliest_present_time_ns: %" PRIu64 // + "\n\tpresent_margin_ns: %" PRIu64 // + "\n\tpresent_margin_ms: %.2fms", // + frame_id, // + f->when_predict_ns, // + f->when_woke_ns, // + f->when_submitted_ns, // + f->when_infoed_ns, // + since_last_frame_ms, // + f->desired_present_time_ns, // + f->actual_present_time_ns, // + f->earliest_present_time_ns, // + f->present_margin_ns, // + present_margin_ms); // + + + if (!U_TRACE_CATEGORY_IS_ENABLED(timing)) { + return; + } + +#define TE_BEG(TRACK, TIME, NAME) U_TRACE_EVENT_BEGIN_ON_TRACK_DATA(timing, TRACK, TIME, NAME, PERCETTO_I(f->frame_id)) +#define TE_END(TRACK, TIME) U_TRACE_EVENT_END_ON_TRACK(timing, TRACK, TIME) + + + /* + * + * CPU + * + */ + + TE_BEG(rt_cpu, f->when_predict_ns, "sleep"); + TE_END(rt_cpu, f->wake_up_time_ns); + + uint64_t oversleep_start_ns = f->wake_up_time_ns + 1; + if (f->when_woke_ns > oversleep_start_ns) { + TE_BEG(rt_cpu, oversleep_start_ns, "oversleep"); + TE_END(rt_cpu, f->when_woke_ns); + } + + + /* + * + * GPU + * + */ + + uint64_t gpu_end_ns = f->actual_present_time_ns - f->present_margin_ns; + if (gpu_end_ns > f->when_submitted_ns) { + TE_BEG(rt_gpu, f->when_submitted_ns, "gpu"); + TE_END(rt_gpu, gpu_end_ns); + } else { + TE_BEG(rt_gpu, gpu_end_ns, "gpu-time-travel"); + TE_END(rt_gpu, f->when_submitted_ns); + } + + + /* + * + * Margin + * + */ + + if (gpu_end_ns < f->desired_present_time_ns) { + TE_BEG(rt_margin, gpu_end_ns, "margin"); + TE_END(rt_margin, f->desired_present_time_ns); + } + + + /* + * + * ERROR + * + */ + + if (!is_within_half_ms(f->actual_present_time_ns, f->desired_present_time_ns)) { + if (f->actual_present_time_ns > f->desired_present_time_ns) { + TE_BEG(rt_error, f->desired_present_time_ns, "slippage"); + TE_END(rt_error, f->actual_present_time_ns); + } else { + TE_BEG(rt_error, f->actual_present_time_ns, "run-ahead"); + TE_END(rt_error, f->desired_present_time_ns); + } + } + + + /* + * + * Info + * + */ + + if (f->when_infoed_ns >= f->actual_present_time_ns) { + TE_BEG(rt_info, f->actual_present_time_ns, "info"); + TE_END(rt_info, f->when_infoed_ns); + } else { + TE_BEG(rt_info, f->when_infoed_ns, "info_before"); + TE_END(rt_info, f->actual_present_time_ns); + } + + + /* + * + * Present + * + */ + + if (f->actual_present_time_ns != f->earliest_present_time_ns) { + U_TRACE_INSTANT_ON_TRACK(timing, rt_present, f->earliest_present_time_ns, "earliest"); + } + if (!is_within_half_ms(f->desired_present_time_ns, f->earliest_present_time_ns)) { + U_TRACE_INSTANT_ON_TRACK(timing, rt_present, f->desired_present_time_ns, "predicted"); + } + U_TRACE_INSTANT_ON_TRACK(timing, rt_present, f->actual_present_time_ns, "vsync"); + + + /* + * + * Compositor time + * + */ + + TE_BEG(rt_allotted, f->wake_up_time_ns, "allotted"); + TE_END(rt_allotted, f->wake_up_time_ns + f->current_app_time_ns); + +#undef TE_BEG +#undef TE_END +} + +static void +dt_destroy(struct u_frame_timing *uft) +{ + struct display_timing *dt = display_timing(uft); + + free(dt); +} + +xrt_result_t +u_ft_display_timing_create(uint64_t estimated_frame_period_ns, struct u_frame_timing **out_uft) +{ + struct display_timing *dt = U_TYPED_CALLOC(struct display_timing); + dt->base.predict = dt_predict; + dt->base.mark_point = dt_mark_point; + dt->base.info = dt_info; + dt->base.destroy = dt_destroy; + dt->frame_period_ns = estimated_frame_period_ns; + + // Just a wild guess. + dt->present_offset_ns = U_TIME_1MS_IN_NS * 4; + + // Start at this of frame time. + dt->app_time_ns = get_percent_of_time(estimated_frame_period_ns, 10); + // Max app time, write a better compositor. + dt->app_time_max_ns = get_percent_of_time(estimated_frame_period_ns, 30); + // When missing, back off in these increments + dt->adjust_missed_ns = get_percent_of_time(estimated_frame_period_ns, 4); + // When not missing frames but adjusting app time at these increments + dt->adjust_non_miss_ns = get_percent_of_time(estimated_frame_period_ns, 2); + // Extra margin that is added to app time. + dt->margin_ns = U_TIME_1MS_IN_NS; + + *out_uft = &dt->base; + + double estimated_frame_period_ms = ns_to_ms(estimated_frame_period_ns); + FT_LOG_I("Created display timing (%.2fms)", estimated_frame_period_ms); + + return XRT_SUCCESS; +} diff --git a/src/xrt/auxiliary/util/u_timing_render.c b/src/xrt/auxiliary/util/u_timing_render.c new file mode 100644 index 000000000..636a63c11 --- /dev/null +++ b/src/xrt/auxiliary/util/u_timing_render.c @@ -0,0 +1,426 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Shared frame timing code. + * @author Jakob Bornecrantz + * @ingroup aux_util + */ + +#include "os/os_time.h" + +#include "util/u_time.h" +#include "util/u_misc.h" +#include "util/u_debug.h" +#include "util/u_timing.h" +#include "util/u_logging.h" +#include "util/u_trace_marker.h" + +#include +#include +#include + + +/* + * + * Structs enums, and defines. + * + */ + +enum u_rt_state +{ + U_RT_READY, + U_RT_WAIT_LEFT, + U_RT_PREDICTED, + U_RT_BEGUN, +}; + +struct u_rt_frame +{ + //! When we predicted this frame to be shown. + uint64_t predicted_display_time_ns; + + //! The selected display period. + uint64_t predicted_display_period_ns; + + //! When the client should have delivered the frame. + uint64_t predicted_delivery_time_ns; + + struct + { + uint64_t predicted_ns; + uint64_t wait_woke_ns; + uint64_t begin_ns; + uint64_t delivered_ns; + } when; //!< When something happened. + + int64_t frame_id; + enum u_rt_state state; +}; + +struct render_timing +{ + struct u_render_timing base; + + struct u_rt_frame frames[2]; + uint32_t current_frame; + uint32_t next_frame; + + int64_t frame_counter; + + struct + { + //! App time between wait returning and begin being called. + uint64_t cpu_time_ns; + //! Time between begin and frame rendering completeing. + uint64_t draw_time_ns; + //! Exrta time between end of draw time and when the compositor wakes up. + uint64_t margin_ns; + } app; //!< App statistics. + + struct + { + //! The last display time that the thing driving this helper got. + uint64_t predicted_display_time_ns; + //! The last display period the hardware is running at. + uint64_t predicted_display_period_ns; + //! The extra time needed by the thing driving this helper. + uint64_t extra_ns; + } last_input; + + uint64_t last_returned_ns; +}; + + +/* + * + * Helpers + * + */ + +static inline struct render_timing * +render_timing(struct u_render_timing *urt) +{ + return (struct render_timing *)urt; +} + +DEBUG_GET_ONCE_LOG_OPTION(ll, "U_TIMING_RENDER_LOG", U_LOGGING_WARN) + +#define RT_LOG_T(...) U_LOG_IFL_T(debug_get_log_option_ll(), __VA_ARGS__) +#define RT_LOG_D(...) U_LOG_IFL_D(debug_get_log_option_ll(), __VA_ARGS__) +#define RT_LOG_I(...) U_LOG_IFL_I(debug_get_log_option_ll(), __VA_ARGS__) +#define RT_LOG_W(...) U_LOG_IFL_W(debug_get_log_option_ll(), __VA_ARGS__) +#define RT_LOG_E(...) U_LOG_IFL_E(debug_get_log_option_ll(), __VA_ARGS__) + +#define DEBUG_PRINT_FRAME_ID() RT_LOG_T("%" PRIi64, frame_id) +#define GET_INDEX_FROM_ID(RT, ID) ((uint64_t)(ID) % ARRAY_SIZE((RT)->frames)) + +#define IIR_ALPHA_LT 0.8 +#define IIR_ALPHA_GT 0.8 + +static void +do_iir_filter(uint64_t *target, double alpha_lt, double alpha_gt, uint64_t sample) +{ + uint64_t t = *target; + double alpha = t < sample ? alpha_lt : alpha_gt; + double a = time_ns_to_s(t) * alpha; + double b = time_ns_to_s(sample) * (1.0 - alpha); + *target = time_s_to_ns(a + b); +} + +static uint64_t +min_period(const struct render_timing *rt) +{ + return rt->last_input.predicted_display_period_ns; +} + +static uint64_t +last_sample_displayed(const struct render_timing *rt) +{ + return rt->last_input.predicted_display_time_ns; +} + +static uint64_t +last_return_predicted_display(const struct render_timing *rt) +{ + return rt->last_returned_ns; +} + +static uint64_t +total_app_time_ns(const struct render_timing *rt) +{ + return rt->app.cpu_time_ns + rt->app.draw_time_ns; +} + +static uint64_t +total_compositor_time_ns(const struct render_timing *rt) +{ + return rt->app.margin_ns + rt->last_input.extra_ns; +} + +static uint64_t +total_app_and_compositor_time_ns(const struct render_timing *rt) +{ + return total_app_time_ns(rt) + total_compositor_time_ns(rt); +} + +static uint64_t +calc_period(const struct render_timing *rt) +{ + // Error checking. + uint64_t base_period_ns = min_period(rt); + if (base_period_ns == 0) { + assert(false && "Have not yet received and samples from timing driver."); + base_period_ns = U_TIME_1MS_IN_NS * 16; // Sure + } + + // Calculate the using both values separately. + uint64_t period_ns = base_period_ns; + while (rt->app.cpu_time_ns > period_ns) { + period_ns += base_period_ns; + } + + while (rt->app.draw_time_ns > period_ns) { + period_ns += base_period_ns; + } + + return period_ns; +} + +static uint64_t +predict_display_time(const struct render_timing *rt, uint64_t period_ns) +{ + // Now + uint64_t now_ns = os_monotonic_get_ns(); + + + // Total app and compositor time to produce a frame + uint64_t app_and_compositor_time_ns = total_app_and_compositor_time_ns(rt); + + // Start from the last time that the driver displayed something. + uint64_t val = last_sample_displayed(rt); + + // Return a time after the last returned display time. + while (val <= last_return_predicted_display(rt)) { + val += period_ns; + } + + // Have to have enough time to perform app work. + while ((val - app_and_compositor_time_ns) <= now_ns) { + val += period_ns; + } + + return val; +} + + +/* + * + * Member functions. + * + */ + +static void +rt_predict(struct u_render_timing *urt, + int64_t *out_frame_id, + uint64_t *out_wake_up_time, + uint64_t *out_predicted_display_time, + uint64_t *out_predicted_display_period) +{ + struct render_timing *rt = render_timing(urt); + + int64_t frame_id = ++rt->frame_counter; + *out_frame_id = frame_id; + + DEBUG_PRINT_FRAME_ID(); + + uint64_t period_ns = calc_period(rt); + uint64_t predict_ns = predict_display_time(rt, period_ns); + // When should the client wake up. + uint64_t wake_up_time_ns = predict_ns - total_app_and_compositor_time_ns(rt); + // When the client should deliver the frame to us. + uint64_t delivery_time_ns = predict_ns - total_compositor_time_ns(rt); + + rt->last_returned_ns = predict_ns; + + *out_wake_up_time = wake_up_time_ns; + *out_predicted_display_time = predict_ns; + *out_predicted_display_period = period_ns; + + size_t index = GET_INDEX_FROM_ID(rt, frame_id); + assert(rt->frames[index].frame_id == -1); + assert(rt->frames[index].state == U_RT_READY); + + rt->frames[index].state = U_RT_PREDICTED; + rt->frames[index].frame_id = frame_id; + rt->frames[index].predicted_delivery_time_ns = delivery_time_ns; + rt->frames[index].predicted_display_period_ns = period_ns; + rt->frames[index].when.predicted_ns = os_monotonic_get_ns(); +} + +static void +rt_mark_point(struct u_render_timing *urt, int64_t frame_id, enum u_timing_point point, uint64_t when_ns) +{ + struct render_timing *rt = render_timing(urt); + + RT_LOG_T("%" PRIi64 " (%u)", frame_id, point); + + size_t index = GET_INDEX_FROM_ID(rt, frame_id); + assert(rt->frames[index].frame_id == frame_id); + + switch (point) { + case U_TIMING_POINT_WAKE_UP: + assert(rt->frames[index].state == U_RT_PREDICTED); + + rt->frames[index].when.wait_woke_ns = when_ns; + rt->frames[index].state = U_RT_WAIT_LEFT; + break; + case U_TIMING_POINT_BEGIN: + assert(rt->frames[index].state == U_RT_WAIT_LEFT); + + rt->frames[index].when.begin_ns = os_monotonic_get_ns(); + rt->frames[index].state = U_RT_BEGUN; + break; + case U_TIMING_POINT_SUBMIT: + default: assert(false); + } +} + +static void +rt_mark_discarded(struct u_render_timing *urt, int64_t frame_id) +{ + struct render_timing *rt = render_timing(urt); + + DEBUG_PRINT_FRAME_ID(); + + size_t index = GET_INDEX_FROM_ID(rt, frame_id); + assert(rt->frames[index].frame_id == frame_id); + assert(rt->frames[index].state == U_RT_WAIT_LEFT || rt->frames[index].state == U_RT_BEGUN); + + rt->frames[index].when.delivered_ns = os_monotonic_get_ns(); + rt->frames[index].state = U_RT_READY; + rt->frames[index].frame_id = -1; +} + +static void +rt_mark_delivered(struct u_render_timing *urt, int64_t frame_id) +{ + struct render_timing *rt = render_timing(urt); + + DEBUG_PRINT_FRAME_ID(); + + size_t index = GET_INDEX_FROM_ID(rt, frame_id); + struct u_rt_frame *f = &rt->frames[index]; + assert(f->frame_id == frame_id); + assert(f->state == U_RT_BEGUN); + + uint64_t now_ns = os_monotonic_get_ns(); + + // Update all data. + f->when.delivered_ns = now_ns; + + + /* + * Process data. + */ + + int64_t diff_ns = f->predicted_delivery_time_ns - now_ns; + bool late = false; + if (diff_ns < 0) { + diff_ns = -diff_ns; + late = true; + } + +#define NS_TO_MS_F(ns) (time_ns_to_s(ns) * 1000.0) + + uint64_t diff_cpu_ns = f->when.begin_ns - f->when.wait_woke_ns; + uint64_t diff_draw_ns = f->when.delivered_ns - f->when.begin_ns; + + RT_LOG_D( + "Delivered frame %.2fms %s." // + "\n\tperiod: %.2f" // + "\n\tcpu o: %.2f, n: %.2f" // + "\n\tdraw o: %.2f, n: %.2f", // + time_ns_to_ms_f(diff_ns), late ? "late" : "early", // + time_ns_to_ms_f(f->predicted_display_period_ns), // + time_ns_to_ms_f(rt->app.cpu_time_ns), time_ns_to_ms_f(diff_cpu_ns), // + time_ns_to_ms_f(rt->app.draw_time_ns), time_ns_to_ms_f(diff_draw_ns)); // + + do_iir_filter(&rt->app.cpu_time_ns, IIR_ALPHA_LT, IIR_ALPHA_GT, diff_cpu_ns); + do_iir_filter(&rt->app.draw_time_ns, IIR_ALPHA_LT, IIR_ALPHA_GT, diff_draw_ns); + + // Trace the data. +#ifdef XRT_FEATURE_TRACING +#define TE_BEG(TRACK, TIME, NAME) U_TRACE_EVENT_BEGIN_ON_TRACK_DATA(timing, TRACK, TIME, NAME, PERCETTO_I(f->frame_id)) +#define TE_END(TRACK, TIME) U_TRACE_EVENT_END_ON_TRACK(timing, TRACK, TIME) + + if (U_TRACE_CATEGORY_IS_ENABLED(timing)) { + TE_BEG(ft_cpu, f->when.predicted_ns, "sleep"); + TE_END(ft_cpu, f->when.wait_woke_ns); + + uint64_t cpu_start_ns = f->when.wait_woke_ns + 1; + TE_BEG(ft_cpu, cpu_start_ns, "cpu"); + TE_END(ft_cpu, f->when.begin_ns); + + TE_BEG(ft_draw, f->when.begin_ns, "draw"); + TE_END(ft_draw, f->when.delivered_ns); + } + +#undef TE_BEG +#undef TE_END +#endif + + // Reset the frame. + f->state = U_RT_READY; + f->frame_id = -1; +} + +static void +rt_info(struct u_render_timing *urt, + uint64_t predicted_display_time_ns, + uint64_t predicted_display_period_ns, + uint64_t extra_ns) +{ + struct render_timing *rt = render_timing(urt); + + rt->last_input.predicted_display_time_ns = predicted_display_time_ns; + rt->last_input.predicted_display_period_ns = predicted_display_period_ns; + rt->last_input.extra_ns = extra_ns; +} + +static void +rt_destroy(struct u_render_timing *urt) +{ + free(urt); +} + + +/* + * + * 'Exported' functions. + * + */ + +xrt_result_t +u_rt_create(struct u_render_timing **out_urt) +{ + struct render_timing *rt = U_TYPED_CALLOC(struct render_timing); + rt->base.predict = rt_predict; + rt->base.mark_point = rt_mark_point; + rt->base.mark_discarded = rt_mark_discarded; + rt->base.mark_delivered = rt_mark_delivered; + rt->base.info = rt_info; + rt->base.destroy = rt_destroy; + rt->app.cpu_time_ns = U_TIME_1MS_IN_NS * 2; + rt->app.draw_time_ns = U_TIME_1MS_IN_NS * 2; + rt->app.margin_ns = U_TIME_1MS_IN_NS * 2; + + for (size_t i = 0; i < ARRAY_SIZE(rt->frames); i++) { + rt->frames[i].state = U_RT_READY; + rt->frames[i].frame_id = -1; + } + + *out_urt = &rt->base; + + return XRT_SUCCESS; +} diff --git a/src/xrt/auxiliary/util/u_trace_marker.c b/src/xrt/auxiliary/util/u_trace_marker.c new file mode 100644 index 000000000..27e8b0bb8 --- /dev/null +++ b/src/xrt/auxiliary/util/u_trace_marker.c @@ -0,0 +1,99 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Tracing support code, see @ref tracing. + * @author Jakob Bornecrantz + * @ingroup aux_util + */ + +#include "xrt/xrt_compiler.h" +#include "xrt/xrt_config_os.h" +#include "xrt/xrt_config_have.h" + +#include "os/os_time.h" + +#include "util/u_trace_marker.h" + +#include + + +#ifdef XRT_FEATURE_TRACING + +PERCETTO_CATEGORY_DEFINE(U_TRACE_CATEGORIES) + +PERCETTO_TRACK_DEFINE(rt_cpu, PERCETTO_TRACK_EVENTS); +PERCETTO_TRACK_DEFINE(rt_allotted, PERCETTO_TRACK_EVENTS); +PERCETTO_TRACK_DEFINE(rt_gpu, PERCETTO_TRACK_EVENTS); +PERCETTO_TRACK_DEFINE(rt_margin, PERCETTO_TRACK_EVENTS); +PERCETTO_TRACK_DEFINE(rt_error, PERCETTO_TRACK_EVENTS); +PERCETTO_TRACK_DEFINE(rt_info, PERCETTO_TRACK_EVENTS); +PERCETTO_TRACK_DEFINE(rt_present, PERCETTO_TRACK_EVENTS); +PERCETTO_TRACK_DEFINE(ft_cpu, PERCETTO_TRACK_EVENTS); +PERCETTO_TRACK_DEFINE(ft_draw, PERCETTO_TRACK_EVENTS); + + +static enum u_trace_which static_which; +static bool static_inited = false; + +void +u_trace_marker_setup(enum u_trace_which which) +{ + static_which = which; + + I_PERCETTO_TRACK_PTR(rt_cpu)->name = "RT 1 Sleep"; + I_PERCETTO_TRACK_PTR(rt_allotted)->name = "RT 2 Allotted time"; + I_PERCETTO_TRACK_PTR(rt_gpu)->name = "RT 3 GPU"; + I_PERCETTO_TRACK_PTR(rt_margin)->name = "RT 4 Margin"; + I_PERCETTO_TRACK_PTR(rt_error)->name = "RT 5 Error"; + I_PERCETTO_TRACK_PTR(rt_info)->name = "RT 6 Info"; + I_PERCETTO_TRACK_PTR(rt_present)->name = "RT 7 Present"; + + I_PERCETTO_TRACK_PTR(ft_cpu)->name = "FT 1 App"; + I_PERCETTO_TRACK_PTR(ft_draw)->name = "FT 2 Draw"; +} + +void +u_trace_marker_init(void) +{ + if (static_inited) { + return; + } + static_inited = true; + + int ret = PERCETTO_INIT(PERCETTO_CLOCK_MONOTONIC); + if (ret != 0) { + return; + } + + if (static_which == U_TRACE_WHICH_SERVICE) { + PERCETTO_REGISTER_TRACK(rt_cpu); + PERCETTO_REGISTER_TRACK(rt_allotted); + PERCETTO_REGISTER_TRACK(rt_gpu); + PERCETTO_REGISTER_TRACK(rt_margin); + PERCETTO_REGISTER_TRACK(rt_error); + PERCETTO_REGISTER_TRACK(rt_info); + PERCETTO_REGISTER_TRACK(rt_present); + + PERCETTO_REGISTER_TRACK(ft_cpu); + PERCETTO_REGISTER_TRACK(ft_draw); + } +} + +#else /* XRT_FEATURE_TRACING */ + +void +u_trace_marker_setup(enum u_trace_which which) +{ + (void)which; + + // Noop +} + +void +u_trace_marker_init(void) +{ + // Noop +} + +#endif /* XRT_FEATURE_TRACING */ diff --git a/src/xrt/auxiliary/util/u_trace_marker.h b/src/xrt/auxiliary/util/u_trace_marker.h new file mode 100644 index 000000000..d3ca65fcc --- /dev/null +++ b/src/xrt/auxiliary/util/u_trace_marker.h @@ -0,0 +1,166 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Tracing support code, see @ref tracing. + * @author Jakob Bornecrantz + * @ingroup aux_util + */ + +#pragma once + +#include "xrt/xrt_compiler.h" +#include "xrt/xrt_config_os.h" +#include "xrt/xrt_config_have.h" +#include "xrt/xrt_config_build.h" + +#include + +#ifdef XRT_FEATURE_TRACING +#include +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * Should the extra tracks be enabled, see @ref tracing. + * + * @ingroup aux_util + */ +enum u_trace_which +{ + U_TRACE_WHICH_SERVICE, + U_TRACE_WHICH_OPENXR, +}; + +/*! + * Internal setup function, use @ref U_TRACE_TARGET_SETUP, see @ref tracing. + * + * @ingroup aux_util + */ +void +u_trace_marker_setup(enum u_trace_which which); + +/*! + * Must be called from a non-static/global constructor context. + * + * @ingroup aux_util + */ +void +u_trace_marker_init(void); + +#define VK_TRACE_IDENT(IDENT) U_TRACE_EVENT(vk, #IDENT) +#define SINK_TRACE_IDENT(IDENT) U_TRACE_EVENT(sink, #IDENT) +#define XRT_TRACE_MARKER() U_TRACE_EVENT(xrt, __func__) +#define IPC_TRACE_MARKER() U_TRACE_EVENT(ipc, __func__) +#define OXR_TRACE_MARKER() U_TRACE_EVENT(oxr, __func__) +#define COMP_TRACE_MARKER() U_TRACE_EVENT(comp, __func__) +#define SINK_TRACE_MARKER() U_TRACE_EVENT(sink, __func__) + + +/* + * + * When enabled. + * + */ + +#ifdef XRT_FEATURE_TRACING + +#ifndef XRT_OS_LINUX +#error "Tracing only supported on Linux" +#endif + +#ifndef XRT_HAVE_PERCETTO +#error "Need to have Percetto/Perfetto" +#endif + + + +#define U_TRACE_CATEGORIES(C, G) \ + C(vk, "vk") /* Vulkan calls */ \ + C(xrt, "xrt") /* Misc XRT calls */ \ + C(sink, "sink") /* Sink/frameserver calls */ \ + C(oxr, "st/oxr") /* OpenXR State Tracker calls */ \ + C(comp, "comp") /* Compositor calls */ \ + C(ipc, "ipc") /* IPC calls */ \ + C(timing, "timing") /* Timing calls */ + +PERCETTO_CATEGORY_DECLARE(U_TRACE_CATEGORIES) + +PERCETTO_TRACK_DECLARE(rt_cpu); +PERCETTO_TRACK_DECLARE(rt_allotted); +PERCETTO_TRACK_DECLARE(rt_gpu); +PERCETTO_TRACK_DECLARE(rt_margin); +PERCETTO_TRACK_DECLARE(rt_error); +PERCETTO_TRACK_DECLARE(rt_info); +PERCETTO_TRACK_DECLARE(rt_present); +PERCETTO_TRACK_DECLARE(ft_cpu); +PERCETTO_TRACK_DECLARE(ft_draw); + +#define U_TRACE_EVENT(CATEGORY, NAME) TRACE_EVENT(CATEGORY, NAME) +#define U_TRACE_EVENT_BEGIN_ON_TRACK(CATEGORY, TRACK, TIME, NAME) \ + TRACE_EVENT_BEGIN_ON_TRACK(CATEGORY, TRACK, TIME, NAME) +#define U_TRACE_EVENT_BEGIN_ON_TRACK_DATA(CATEGORY, TRACK, TIME, NAME, ...) \ + TRACE_EVENT_BEGIN_ON_TRACK_DATA(CATEGORY, TRACK, TIME, NAME, __VA_ARGS__) +#define U_TRACE_EVENT_END_ON_TRACK(CATEGORY, TRACK, TIME) TRACE_EVENT_END_ON_TRACK(CATEGORY, TRACK, TIME) +#define U_TRACE_CATEGORY_IS_ENABLED(CATEGORY) PERCETTO_CATEGORY_IS_ENABLED(CATEGORY) +#define U_TRACE_INSTANT_ON_TRACK(CATEGORY, TRACK, TIME, NAME) \ + TRACE_ANY_WITH_ARGS(PERCETTO_EVENT_INSTANT, CATEGORY, &g_percetto_track_##TRACK, TIME, NAME, 0) +#define U_TRACE_DATA(fd, type, data) u_trace_data(fd, type, (void *)&(data), sizeof(data)) + +#define U_TRACE_TARGET_SETUP(WHICH) \ + void __attribute__((constructor(101))) u_trace_marker_constructor(void); \ + \ + void u_trace_marker_constructor(void) \ + { \ + u_trace_marker_setup(WHICH); \ + } + + +#else // XRT_FEATURE_TRACING + + +/* + * + * When disabled. + * + */ + +#define U_TRACE_EVENT(CATEGORY, NAME) \ + do { \ + } while (false) + +#define U_TRACE_EVENT_BEGIN_ON_TRACK(CATEGORY, TRACK, TIME, NAME) \ + do { \ + } while (false) + +#define U_TRACE_EVENT_BEGIN_ON_TRACK_DATA(CATEGORY, TRACK, TIME, NAME, ...) \ + do { \ + } while (false) + +#define U_TRACE_EVENT_END_ON_TRACK(CATEGORY, TRACK, TIME) \ + do { \ + } while (false) + +#define U_TRACE_INSTANT_ON_TRACK(CATEGORY, TRACK, TIME, NAME) \ + do { \ + } while (false) + +#define U_TRACE_CATEGORY_IS_ENABLED(_) (false) + +/*! + * Add to target c file to enable tracing, see @ref tracing. + * + * @ingroup aux_util + */ +#define U_TRACE_TARGET_SETUP(WHICH) + +#endif // XRT_FEATURE_TRACING + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/util/u_var.cpp b/src/xrt/auxiliary/util/u_var.cpp index 890e8276f..6b759d46d 100644 --- a/src/xrt/auxiliary/util/u_var.cpp +++ b/src/xrt/auxiliary/util/u_var.cpp @@ -52,7 +52,7 @@ public: getNumber(const std::string &name) { auto s = counters.find(name); - int count = (s != counters.end() ? s->second : 0) + 1; + int count = int(s != counters.end() ? s->second : 0) + 1; counters[name] = count; return count; @@ -89,7 +89,7 @@ add_var(void *root, void *ptr, u_var_kind kind, const char *c_name) } Var var; - snprintf(var.info.name, 256, "%s", c_name); + snprintf(var.info.name, U_VAR_NAME_STRING_SIZE, "%s", c_name); var.info.kind = kind; var.info.ptr = ptr; diff --git a/src/xrt/auxiliary/util/u_var.h b/src/xrt/auxiliary/util/u_var.h index 1915a8099..cf0cba443 100644 --- a/src/xrt/auxiliary/util/u_var.h +++ b/src/xrt/auxiliary/util/u_var.h @@ -20,11 +20,12 @@ extern "C" { struct xrt_frame_sink; +struct u_sink_debug; struct m_ff_f64; struct m_ff_vec3_f32; /*! - * @ingroup aux_util + * @addtogroup aux_util * @{ */ @@ -56,6 +57,38 @@ struct u_var_timing const char *unit; }; +/*! + * Callback for a button action + */ +typedef void (*u_var_button_cb)(void *); + +struct u_var_button +{ + //! Callback function to execute on button press + u_var_button_cb cb; + + //! Pointer that will be passed to the function as its only argument + void *ptr; + + //! Button text, use var `name` if zeroed + char label[64]; + + //! Button dimensions, zero for auto size + float width; + float height; + + //! Whether this button is disabled + bool disabled; +}; + +struct u_var_draggable_f32 +{ + float val; + float step; + float min; + float max; +}; + /*! * What kind of variable is this tracking. */ @@ -65,14 +98,17 @@ enum u_var_kind U_VAR_KIND_RGB_U8, U_VAR_KIND_RGB_F32, U_VAR_KIND_U8, + U_VAR_KIND_U64, U_VAR_KIND_I32, U_VAR_KIND_F32, + U_VAR_KIND_DRAGGABLE_F32, + U_VAR_KIND_F64, U_VAR_KIND_F32_ARR, U_VAR_KIND_TIMING, U_VAR_KIND_VEC3_I32, U_VAR_KIND_VEC3_F32, U_VAR_KIND_POSE, - U_VAR_KIND_SINK, + U_VAR_KIND_SINK_DEBUG, U_VAR_KIND_LOG_LEVEL, U_VAR_KIND_RO_TEXT, U_VAR_KIND_RO_I32, @@ -87,15 +123,17 @@ enum u_var_kind U_VAR_KIND_RO_FF_F64, U_VAR_KIND_RO_FF_VEC3_F32, U_VAR_KIND_GUI_HEADER, + U_VAR_KIND_BUTTON, }; +#define U_VAR_NAME_STRING_SIZE 256 /*! * Struct that keeps all of the information about the variable, some of the UI * state is kept on it. */ struct u_var_info { - char name[256]; + char name[U_VAR_NAME_STRING_SIZE]; void *ptr; enum u_var_kind kind; @@ -166,14 +204,16 @@ u_var_force_on(void); ADD_FUNC(rgb_u8, struct xrt_colour_rgb_u8, RGB_U8) \ ADD_FUNC(rgb_f32, struct xrt_colour_rgb_f32, RGB_F32) \ ADD_FUNC(u8, uint8_t, U8) \ + ADD_FUNC(u64, uint64_t, U64) \ ADD_FUNC(i32, int32_t, I32) \ ADD_FUNC(f32, float, F32) \ + ADD_FUNC(f64, double, F64) \ ADD_FUNC(f32_arr, struct u_var_f32_arr, F32_ARR) \ ADD_FUNC(f32_timing, struct u_var_timing, TIMING) \ ADD_FUNC(vec3_i32, struct xrt_vec3_i32, VEC3_I32) \ ADD_FUNC(vec3_f32, struct xrt_vec3, VEC3_F32) \ ADD_FUNC(pose, struct xrt_pose, POSE) \ - ADD_FUNC(sink, struct xrt_frame_sink *, SINK) \ + ADD_FUNC(sink_debug, struct u_sink_debug, SINK_DEBUG) \ ADD_FUNC(log_level, enum u_logging_level, LOG_LEVEL) \ ADD_FUNC(ro_text, const char, RO_TEXT) \ ADD_FUNC(ro_i32, int32_t, RO_I32) \ @@ -187,7 +227,9 @@ u_var_force_on(void); ADD_FUNC(ro_quat_f32, struct xrt_quat, RO_QUAT_F32) \ ADD_FUNC(ro_ff_f64, struct m_ff_f64, RO_FF_F64) \ ADD_FUNC(ro_ff_vec3_f32, struct m_ff_vec3_f32, RO_FF_VEC3_F32) \ - ADD_FUNC(gui_header, bool, GUI_HEADER) + ADD_FUNC(gui_header, bool, GUI_HEADER) \ + ADD_FUNC(button, struct u_var_button, BUTTON) \ + ADD_FUNC(draggable_f32, struct u_var_draggable_f32, DRAGGABLE_F32) #define ADD_FUNC(SUFFIX, TYPE, ENUM) void u_var_add_##SUFFIX(void *, TYPE *, const char *); diff --git a/src/xrt/auxiliary/util/u_verify.h b/src/xrt/auxiliary/util/u_verify.h new file mode 100644 index 000000000..5270f9c3c --- /dev/null +++ b/src/xrt/auxiliary/util/u_verify.h @@ -0,0 +1,32 @@ +// Copyright 2021, Collabora, Ltd. +// Copyright 2021, Moses Turner. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Tiny file to verify things + * @author Moses Turner + * @author Jakob Bornecrantz + * @ingroup aux_util + */ + +#pragma once +#include "xrt/xrt_defines.h" +#include "xrt/xrt_device.h" + +static inline bool +u_verify_blend_mode_valid(enum xrt_blend_mode blend_mode) +{ + return ((blend_mode == XRT_BLEND_MODE_OPAQUE) || (blend_mode == XRT_BLEND_MODE_ADDITIVE) || + (blend_mode == XRT_BLEND_MODE_ALPHA_BLEND)); +} + +static inline bool +u_verify_blend_mode_supported(struct xrt_device *xdev, enum xrt_blend_mode blend_mode) +{ + for (size_t i = 0; i < xdev->hmd->num_blend_modes; i++) { + if (xdev->hmd->blend_modes[i] == blend_mode) { + return true; + } + } + return false; +} diff --git a/src/xrt/drivers/vive/vive_config.c b/src/xrt/auxiliary/vive/vive_config.c similarity index 54% rename from src/xrt/drivers/vive/vive_config.c rename to src/xrt/auxiliary/vive/vive_config.c index c7e58b3d1..f7319880d 100644 --- a/src/xrt/drivers/vive/vive_config.c +++ b/src/xrt/auxiliary/vive/vive_config.c @@ -1,9 +1,10 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Vive json implementation * @author Lubosz Sarnecki + * @author Moses Turner * @ingroup drv_vive */ @@ -11,14 +12,22 @@ #include "vive_config.h" +#include "util/u_misc.h" #include "util/u_json.h" #include "util/u_distortion_mesh.h" #include "math/m_api.h" -#include "vive.h" -#include "vive_device.h" -#include "vive_controller.h" +#include "tracking/t_tracking.h" +#include "math/m_vec3.h" +#include "math/m_space.h" + + +#define VIVE_TRACE(d, ...) U_LOG_IFL_T(d->ll, __VA_ARGS__) +#define VIVE_DEBUG(d, ...) U_LOG_IFL_D(d->ll, __VA_ARGS__) +#define VIVE_INFO(d, ...) U_LOG_IFL_I(d->ll, __VA_ARGS__) +#define VIVE_WARN(d, ...) U_LOG_IFL_W(d->ll, __VA_ARGS__) +#define VIVE_ERROR(d, ...) U_LOG_IFL_E(d->ll, __VA_ARGS__) #define JSON_INT(a, b, c) u_json_get_int(u_json_get(a, b), c) #define JSON_FLOAT(a, b, c) u_json_get_float(u_json_get(a, b), c) @@ -27,6 +36,10 @@ #define JSON_MATRIX_3X3(a, b, c) u_json_get_matrix_3x3(u_json_get(a, b), c) #define JSON_STRING(a, b, c) u_json_get_string_into_array(u_json_get(a, b), c, sizeof(c)) +#define printf_pose(pose) \ + printf("%f %f %f %f %f %f %f\n", pose.position.x, pose.position.y, pose.position.z, pose.orientation.x, \ + pose.orientation.y, pose.orientation.z, pose.orientation.w); + static void _get_color_coeffs(struct u_vive_values *values, const cJSON *coeffs, uint8_t eye, uint8_t channel) { @@ -56,7 +69,7 @@ _get_pose_from_pos_x_z(const cJSON *obj, struct xrt_pose *pose) } static void -_get_distortion_properties(struct vive_device *d, const cJSON *eye_transform_json, uint8_t eye) +_get_distortion_properties(struct vive_config *d, const cJSON *eye_transform_json, uint8_t eye) { const cJSON *eye_json = cJSON_GetArrayItem(eye_transform_json, eye); if (eye_json == NULL) { @@ -97,7 +110,7 @@ _get_distortion_properties(struct vive_device *d, const cJSON *eye_transform_jso } static void -_get_lighthouse(struct vive_device *d, const cJSON *json) +_get_lighthouse(struct vive_config *d, const cJSON *json) { const cJSON *lh = cJSON_GetObjectItemCaseSensitive(json, "lighthouse_config"); if (lh == NULL) { @@ -158,7 +171,7 @@ _get_lighthouse(struct vive_device *d, const cJSON *json) // Transform the sensors into IMU space. - struct xrt_pose trackref_to_imu = {0}; + struct xrt_pose trackref_to_imu = XRT_POSE_IDENTITY; math_pose_invert(&d->imu.trackref, &trackref_to_imu); for (i = 0; i < d->lh.num_sensors; i++) { @@ -176,17 +189,235 @@ _print_vec3(const char *title, struct xrt_vec3 *vec) U_LOG_D("%s = %f %f %f", title, (double)vec->x, (double)vec->y, (double)vec->z); } -bool -vive_config_parse(struct vive_device *d, char *json_string) +static bool +_get_camera(struct index_camera *cam, const cJSON *cam_json) { + bool succeeded = true; + const cJSON *extrinsics = u_json_get(cam_json, "extrinsics"); + _get_pose_from_pos_x_z(extrinsics, &cam->trackref); + + + const cJSON *intrinsics = u_json_get(cam_json, "intrinsics"); + + succeeded = succeeded && u_json_get_double_array(u_json_get(u_json_get(intrinsics, "distort"), "coeffs"), + cam->intrinsics.distortion, 4); + + succeeded = succeeded && u_json_get_double(u_json_get(intrinsics, "center_x"), &cam->intrinsics.center_x); + succeeded = succeeded && u_json_get_double(u_json_get(intrinsics, "center_y"), &cam->intrinsics.center_y); + + succeeded = succeeded && u_json_get_double(u_json_get(intrinsics, "focal_x"), &cam->intrinsics.focal_x); + succeeded = succeeded && u_json_get_double(u_json_get(intrinsics, "focal_y"), &cam->intrinsics.focal_y); + succeeded = succeeded && u_json_get_int(u_json_get(intrinsics, "height"), &cam->intrinsics.image_size_pixels.h); + succeeded = succeeded && u_json_get_int(u_json_get(intrinsics, "width"), &cam->intrinsics.image_size_pixels.w); + + if (!succeeded) { + return false; + } + return true; +} + +static bool +_get_cameras(struct vive_config *d, const cJSON *cameras_json) +{ + const cJSON *cmr = NULL; + + bool found_camera_json = false; + bool succeeded_parsing_json = false; + + cJSON_ArrayForEach(cmr, cameras_json) + { + found_camera_json = true; + + const cJSON *name_json = u_json_get(cmr, "name"); + const char *name = name_json->valuestring; + bool is_left = !strcmp("left", name); + bool is_right = !strcmp("right", name); + + if (!is_left && !is_right) { + continue; + } + + if (!_get_camera(&d->cameras.view[is_right], cmr)) { + succeeded_parsing_json = false; + break; + } + + succeeded_parsing_json = true; + } + + if (!found_camera_json) { + U_LOG_W("HMD is Index, but no cameras in json file!"); + return false; + } else if (!succeeded_parsing_json) { + U_LOG_E("Failed to parse Index camera calibration!"); + return false; + } + + struct xrt_pose trackref_to_head; + struct xrt_pose camera_to_head; + math_pose_invert(&d->display.trackref, &trackref_to_head); + + math_pose_transform(&trackref_to_head, &d->cameras.view[0].trackref, &camera_to_head); + d->cameras.view[0].headref = camera_to_head; + + math_pose_transform(&trackref_to_head, &d->cameras.view[1].trackref, &camera_to_head); + d->cameras.view[1].headref = camera_to_head; + + // Calculate where in the right camera space the left camera is. + struct xrt_pose invert, left_in_right; + math_pose_invert(&d->cameras.view[1].headref, &invert); + math_pose_transform(&d->cameras.view[0].headref, &invert, &left_in_right); + d->cameras.left_in_right = left_in_right; + + // To turn it into OpenCV cameras coordinate system. + struct xrt_pose opencv = left_in_right; + opencv.orientation.x = -left_in_right.orientation.x; + opencv.position.y = -left_in_right.position.y; + opencv.position.z = -left_in_right.position.z; + d->cameras.opencv = opencv; + + d->cameras.valid = true; + + return true; +} + +bool +vive_get_stereo_camera_calibration(struct vive_config *d, + struct t_stereo_camera_calibration **calibration_ptr_to_ref, + struct xrt_pose *out_head_in_left_camera) +{ + if (!d->cameras.valid) { + U_LOG_E("Camera config not loaded, can not produce camera calibration."); + return false; + } + + struct index_camera *cameras = d->cameras.view; + struct t_stereo_camera_calibration *calib = NULL; + + t_stereo_camera_calibration_alloc(&calib, 5); + + for (int i = 0; i < 2; i++) { + calib->view[i].image_size_pixels.w = cameras[i].intrinsics.image_size_pixels.w; + calib->view[i].image_size_pixels.h = cameras[i].intrinsics.image_size_pixels.h; + + // This better be row-major! + calib->view[i].intrinsics[0][0] = cameras[i].intrinsics.focal_x; + calib->view[i].intrinsics[0][1] = 0.0f; + calib->view[i].intrinsics[0][2] = cameras[i].intrinsics.center_x; + + calib->view[i].intrinsics[1][0] = 0.0f; + calib->view[i].intrinsics[1][1] = cameras[i].intrinsics.focal_y; + calib->view[i].intrinsics[1][2] = cameras[i].intrinsics.center_y; + + calib->view[i].intrinsics[2][0] = 0.0f; + calib->view[i].intrinsics[2][1] = 0.0f; + calib->view[i].intrinsics[2][2] = 1.0f; + + calib->view[i].use_fisheye = true; + calib->view[i].distortion_fisheye[0] = cameras[i].intrinsics.distortion[0]; + calib->view[i].distortion_fisheye[1] = cameras[i].intrinsics.distortion[1]; + calib->view[i].distortion_fisheye[2] = cameras[i].intrinsics.distortion[2]; + calib->view[i].distortion_fisheye[3] = cameras[i].intrinsics.distortion[3]; + } + + struct xrt_vec3 pos = d->cameras.opencv.position; + struct xrt_vec3 x = {1, 0, 0}, y = {0, 1, 0}, z = {0, 0, 1}; + math_quat_rotate_vec3(&d->cameras.opencv.orientation, &x, &x); + math_quat_rotate_vec3(&d->cameras.opencv.orientation, &y, &y); + math_quat_rotate_vec3(&d->cameras.opencv.orientation, &z, &z); + + calib->camera_translation[0] = pos.x; + calib->camera_translation[1] = pos.y; + calib->camera_translation[2] = pos.z; + + calib->camera_rotation[0][0] = x.x; + calib->camera_rotation[0][1] = x.y; + calib->camera_rotation[0][2] = x.z; + + calib->camera_rotation[1][0] = y.x; + calib->camera_rotation[1][1] = y.y; + calib->camera_rotation[1][2] = y.z; + + calib->camera_rotation[2][0] = z.x; + calib->camera_rotation[2][1] = z.y; + calib->camera_rotation[2][2] = z.z; + + math_pose_invert(&d->cameras.view[0].headref, out_head_in_left_camera); + + // Correctly reference count. + t_stereo_camera_calibration_reference(calibration_ptr_to_ref, calib); + t_stereo_camera_calibration_reference(&calib, NULL); + + return true; +} + +static void +vive_init_defaults(struct vive_config *d) +{ + d->display.eye_target_width_in_pixels = 1080; + d->display.eye_target_height_in_pixels = 1200; + + d->display.rot[0].w = 1.0f; + d->display.rot[1].w = 1.0f; + + d->imu.gyro_range = 8.726646f; + d->imu.acc_range = 39.226600f; + + d->imu.acc_scale.x = 1.0f; + d->imu.acc_scale.y = 1.0f; + d->imu.acc_scale.z = 1.0f; + + d->imu.gyro_scale.x = 1.0f; + d->imu.gyro_scale.y = 1.0f; + d->imu.gyro_scale.z = 1.0f; + + d->cameras.valid = false; + + for (int view = 0; view < 2; view++) { + d->distortion[view].aspect_x_over_y = 0.89999997615814209f; + d->distortion[view].grow_for_undistort = 0.5f; + d->distortion[view].undistort_r2_cutoff = 1.0f; + } +} + +bool +vive_config_parse(struct vive_config *d, char *json_string, enum u_logging_level ll) +{ + d->ll = ll; + vive_init_defaults(d); + VIVE_DEBUG(d, "JSON config:\n%s", json_string); cJSON *json = cJSON_Parse(json_string); if (!cJSON_IsObject(json)) { VIVE_ERROR(d, "Could not parse JSON data."); + vive_config_teardown(d); return false; } + if (u_json_get(json, "model_number")) { + JSON_STRING(json, "model_number", d->firmware.model_number); + } else { + JSON_STRING(json, "model_name", d->firmware.model_number); + } + + VIVE_DEBUG(d, "Parsing model number: %s", d->firmware.model_number); + + if (strcmp(d->firmware.model_number, "Utah MP") == 0) { + d->variant = VIVE_VARIANT_INDEX; + VIVE_DEBUG(d, "Found Valve Index HMD"); + } else if (strcmp(d->firmware.model_number, "Vive MV") == 0 || + strcmp(d->firmware.model_number, "Vive MV.") == 0 || + strcmp(d->firmware.model_number, "Vive. MV") == 0) { + d->variant = VIVE_VARIANT_VIVE; + VIVE_DEBUG(d, "Found HTC Vive HMD"); + } else if (strcmp(d->firmware.model_number, "Vive_Pro MV") == 0) { + d->variant = VIVE_VARIANT_PRO; + VIVE_DEBUG(d, "Found HTC Vive Pro HMD"); + } else { + VIVE_ERROR(d, "Failed to parse Vive HMD variant"); + } + switch (d->variant) { case VIVE_VARIANT_VIVE: JSON_VEC3(json, "acc_bias", &d->imu.acc_bias); @@ -221,11 +452,15 @@ vive_config_parse(struct vive_device *d, char *json_string) math_pose_transform(&trackref_to_head, &d->imu.trackref, &imu_to_head); d->display.imuref = imu_to_head; - } break; - default: VIVE_ERROR(d, "Unknown Vive variant."); return false; - } - JSON_STRING(json, "model_number", d->firmware.model_number); + const cJSON *cameras_json = u_json_get(json, "tracked_cameras"); + _get_cameras(d, cameras_json); + } break; + default: + VIVE_ERROR(d, "Unknown Vive variant."); + vive_config_teardown(d); + return false; + } if (d->variant != VIVE_VARIANT_INDEX) { JSON_STRING(json, "mb_serial_number", d->firmware.mb_serial_number); @@ -270,7 +505,7 @@ vive_config_parse(struct vive_device *d, char *json_string) VIVE_DEBUG(d, "eye_target_height_in_pixels: %d", d->display.eye_target_height_in_pixels); VIVE_DEBUG(d, "eye_target_width_in_pixels: %d", d->display.eye_target_width_in_pixels); - if (d->ll >= U_LOGGING_DEBUG) { + if (d->ll <= U_LOGGING_DEBUG) { _print_vec3("acc_bias", &d->imu.acc_bias); _print_vec3("acc_scale", &d->imu.acc_scale); _print_vec3("gyro_bias", &d->imu.gyro_bias); @@ -286,9 +521,20 @@ vive_config_parse(struct vive_device *d, char *json_string) return true; } -bool -vive_config_parse_controller(struct vive_controller_device *d, char *json_string) +void +vive_config_teardown(struct vive_config *config) { + if (config->lh.sensors != NULL) { + free(config->lh.sensors); + config->lh.sensors = NULL; + config->lh.num_sensors = 0; + } +} + +bool +vive_config_parse_controller(struct vive_controller_config *d, char *json_string, enum u_logging_level ll) +{ + d->ll = ll; VIVE_DEBUG(d, "JSON config:\n%s", json_string); cJSON *json = cJSON_Parse(json_string); @@ -304,16 +550,23 @@ vive_config_parse_controller(struct vive_controller_device *d, char *json_string JSON_STRING(json, "model_name", d->firmware.model_number); } - if (strcmp(d->firmware.model_number, "Vive. Controller MV") == 0) { + VIVE_DEBUG(d, "Parsing model number: %s", d->firmware.model_number); + + if (strcmp(d->firmware.model_number, "Vive. Controller MV") == 0 || + strcmp(d->firmware.model_number, "Vive Controller MV") == 0) { d->variant = CONTROLLER_VIVE_WAND; VIVE_DEBUG(d, "Found Vive Wand controller"); - } else if (strcmp(d->firmware.model_number, "Knuckles Right") == 0) { + } else if (strcmp(d->firmware.model_number, "Knuckles Right") == 0 || + strcmp(d->firmware.model_number, "Knuckles EV3.0 Right") == 0) { d->variant = CONTROLLER_INDEX_RIGHT; VIVE_DEBUG(d, "Found Knuckles Right controller"); - } else if (strcmp(d->firmware.model_number, "Knuckles Left") == 0) { + } else if (strcmp(d->firmware.model_number, "Knuckles Left") == 0 || + strcmp(d->firmware.model_number, "Knuckles EV3.0 Left") == 0) { d->variant = CONTROLLER_INDEX_LEFT; VIVE_DEBUG(d, "Found Knuckles Left controller"); - } else if (strcmp(d->firmware.model_number, "Vive Tracker PVT") == 0) { + } else if (strcmp(d->firmware.model_number, "Vive Tracker PVT") == 0 || + strcmp(d->firmware.model_number, "Vive. Tracker MV") == 0 || + strcmp(d->firmware.model_number, "Vive Tracker MV") == 0) { d->variant = CONTROLLER_TRACKER_GEN1; VIVE_DEBUG(d, "Found Gen 1 tracker."); } else if (strcmp(d->firmware.model_number, "VIVE Tracker Pro MV") == 0) { diff --git a/src/xrt/auxiliary/vive/vive_config.h b/src/xrt/auxiliary/vive/vive_config.h new file mode 100644 index 000000000..125cb02ee --- /dev/null +++ b/src/xrt/auxiliary/vive/vive_config.h @@ -0,0 +1,222 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief vive json header + * @author Lubosz Sarnecki + * @author Moses Turner + * @ingroup drv_vive + */ + +#pragma once + +#include + +#include "xrt/xrt_defines.h" +#include "util/u_logging.h" +#include "util/u_distortion_mesh.h" +#include "tracking/t_tracking.h" + +// public documentation +#define INDEX_MIN_IPD 0.058 +#define INDEX_MAX_IPD 0.07 + +// arbitrary default values +#define DEFAULT_HAPTIC_FREQ 150.0f +#define MIN_HAPTIC_DURATION 0.05f + +enum VIVE_VARIANT +{ + VIVE_UNKNOWN = 0, + VIVE_VARIANT_VIVE, + VIVE_VARIANT_PRO, + VIVE_VARIANT_INDEX +}; + +enum VIVE_CONTROLLER_VARIANT +{ + CONTROLLER_VIVE_WAND, + CONTROLLER_INDEX_LEFT, + CONTROLLER_INDEX_RIGHT, + CONTROLLER_TRACKER_GEN1, + CONTROLLER_TRACKER_GEN2, + CONTROLLER_UNKNOWN +}; + +/*! + * A calibrated camera on an Index. + */ +struct index_camera +{ + // Note! All the values in this struct are directly pasted in from the JSON values. + // As such, in my opinion, plus_x, plus_z and position are all "wrong" - all the code I've had to write that + // uses this struct flips the signs of plus_x, plus_z, and the signs of the X- and Z-components of position. + // I have no idea why those sign flips were necessary - I suppose Valve/HTC just made some weird decisions when + // making the config file schemas. I figure it would be very confusing to try to "fix" these values as I'm + // parsing them, so if you're writing code downstream of this, beware and expect the values in here to be + // exactly the same as those in the compressed JSON. -Moses + struct + { + struct xrt_vec3 plus_x; + struct xrt_vec3 plus_z; + struct xrt_vec3 position; // looks like from head pose + } extrinsics; + + //! Pose in tracking space. + struct xrt_pose trackref; + + //! Pose in head space. + struct xrt_pose headref; + + struct + { + double distortion[4]; // Kannala-Brandt + double center_x; + double center_y; + + double focal_x; + double focal_y; + struct xrt_size image_size_pixels; + } intrinsics; +}; + +/*! + * A single lighthouse senosor point and normal, in IMU space. + */ +struct lh_sensor +{ + struct xrt_vec3 pos; + uint32_t _pad0; + struct xrt_vec3 normal; + uint32_t _pad1; +}; + +/*! + * A lighthouse consisting of sensors. + * + * All sensors are placed in IMU space. + */ +struct lh_model +{ + struct lh_sensor *sensors; + size_t num_sensors; +}; + +struct vive_config +{ + //! log level accessed by the config parser + enum u_logging_level ll; + + enum VIVE_VARIANT variant; + + struct + { + double acc_range; + double gyro_range; + struct xrt_vec3 acc_bias; + struct xrt_vec3 acc_scale; + struct xrt_vec3 gyro_bias; + struct xrt_vec3 gyro_scale; + + //! IMU position in tracking space. + struct xrt_pose trackref; + } imu; + + struct + { + double lens_separation; + double persistence; + int eye_target_height_in_pixels; + int eye_target_width_in_pixels; + + struct xrt_quat rot[2]; + + //! Head position in tracking space. + struct xrt_pose trackref; + //! Head position in IMU space. + struct xrt_pose imuref; + } display; + + struct + { + uint32_t display_firmware_version; + uint32_t firmware_version; + uint8_t hardware_revision; + uint8_t hardware_version_micro; + uint8_t hardware_version_minor; + uint8_t hardware_version_major; + char mb_serial_number[32]; + char model_number[32]; + char device_serial_number[32]; + } firmware; + + struct u_vive_values distortion[2]; + + struct + { + //! The two cameras. + struct index_camera view[2]; + + //! Left view in right camera space. + struct xrt_pose left_in_right; + + //! The same but in OpenCV camera space. + struct xrt_pose opencv; + + //! Have we loaded the config. + bool valid; + } cameras; + + struct lh_model lh; +}; + +struct vive_controller_config +{ + enum u_logging_level ll; + + enum VIVE_CONTROLLER_VARIANT variant; + + struct + { + uint32_t firmware_version; + uint8_t hardware_revision; + uint8_t hardware_version_micro; + uint8_t hardware_version_minor; + uint8_t hardware_version_major; + char mb_serial_number[32]; + char model_number[32]; + char device_serial_number[32]; + } firmware; + + struct + { + double acc_range; + double gyro_range; + struct xrt_vec3 acc_bias; + struct xrt_vec3 acc_scale; + struct xrt_vec3 gyro_bias; + struct xrt_vec3 gyro_scale; + + //! IMU position in tracking space. + struct xrt_pose trackref; + } imu; +}; + +bool +vive_config_parse(struct vive_config *d, char *json_string, enum u_logging_level ll); + +/*! + * Free any allocated resources on this config. + */ +void +vive_config_teardown(struct vive_config *d); + +struct vive_controller_device; + +bool +vive_config_parse_controller(struct vive_controller_config *d, char *json_string, enum u_logging_level ll); + +bool +vive_get_stereo_camera_calibration(struct vive_config *d, + struct t_stereo_camera_calibration **calibration_ptr_to_ref, + struct xrt_pose *out_head_in_left_camera); diff --git a/src/xrt/auxiliary/vk/vk_helpers.c b/src/xrt/auxiliary/vk/vk_helpers.c index 35820fb98..f57869717 100644 --- a/src/xrt/auxiliary/vk/vk_helpers.c +++ b/src/xrt/auxiliary/vk/vk_helpers.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -58,6 +58,7 @@ vk_result_string(VkResult code) ENUM_TO_STR(VK_ERROR_VALIDATION_FAILED_EXT); ENUM_TO_STR(VK_ERROR_INVALID_SHADER_NV); ENUM_TO_STR(VK_ERROR_INVALID_EXTERNAL_HANDLE); + ENUM_TO_STR(VK_ERROR_NOT_PERMITTED_EXT); default: return "UNKNOWN RESULT"; } } @@ -81,6 +82,7 @@ vk_color_format_string(VkFormat code) ENUM_TO_STR(VK_FORMAT_D16_UNORM); ENUM_TO_STR(VK_FORMAT_A2B10G10R10_UNORM_PACK32); ENUM_TO_STR(VK_FORMAT_R16G16B16A16_SFLOAT); + ENUM_TO_STR(VK_FORMAT_R16G16B16A16_UNORM); ENUM_TO_STR(VK_FORMAT_R8G8B8A8_UNORM); default: return "UNKNOWN FORMAT"; } @@ -170,19 +172,33 @@ bool vk_get_memory_type(struct vk_bundle *vk, uint32_t type_bits, VkMemoryPropertyFlags memory_props, uint32_t *out_type_id) { + uint32_t i_supported = type_bits; for (uint32_t i = 0; i < vk->device_memory_props.memoryTypeCount; i++) { uint32_t propertyFlags = vk->device_memory_props.memoryTypes[i].propertyFlags; - if ((type_bits & 1) == 1) { + if ((i_supported & 1) == 1) { if ((propertyFlags & memory_props) == memory_props) { *out_type_id = i; return true; } } - type_bits >>= 1; + i_supported >>= 1; } VK_DEBUG(vk, "Could not find memory type!"); + VK_TRACE(vk, "Requested flags: %d (type bits %d with %d memory types)", memory_props, type_bits, + vk->device_memory_props.memoryTypeCount); + + i_supported = type_bits; + VK_TRACE(vk, "Supported flags:"); + for (uint32_t i = 0; i < vk->device_memory_props.memoryTypeCount; i++) { + uint32_t propertyFlags = vk->device_memory_props.memoryTypes[i].propertyFlags; + if ((i_supported & 1) == 1) { + VK_TRACE(vk, " %d", propertyFlags); + } + i_supported >>= 1; + } + return false; } @@ -198,9 +214,7 @@ vk_alloc_and_bind_image_memory(struct vk_bundle *vk, vk->vkGetImageMemoryRequirements(vk->device, image, &memory_requirements); if (max_size > 0 && memory_requirements.size > max_size) { - VK_ERROR(vk, - "client_vk_swapchain - Got too little memory " - "%u vs %u\n", + VK_ERROR(vk, "client_vk_swapchain - Got too little memory %u vs %u\n", (uint32_t)memory_requirements.size, (uint32_t)max_size); return VK_ERROR_OUT_OF_DEVICE_MEMORY; } @@ -299,9 +313,7 @@ vk_create_image_from_native(struct vk_bundle *vk, { VkImageUsageFlags image_usage = vk_swapchain_usage_flags(vk, (VkFormat)info->format, info->bits); if (image_usage == 0) { - U_LOG_E( - "vk_create_image_from_native: Unsupported swapchain usage " - "flags"); + U_LOG_E("vk_create_image_from_native: Unsupported swapchain usage flags"); return VK_ERROR_FEATURE_NOT_PRESENT; } @@ -327,6 +339,44 @@ vk_create_image_from_native(struct vk_bundle *vk, #error "need port" #endif + VkPhysicalDeviceExternalImageFormatInfo external_image_format_info = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO, + .handleType = external_memory_image_create_info.handleTypes, + }; + + VkPhysicalDeviceImageFormatInfo2 format_info = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, + .pNext = &external_image_format_info, + .format = (VkFormat)info->format, + .type = VK_IMAGE_TYPE_2D, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = image_usage, + }; + + VkExternalImageFormatProperties external_format_properties = { + .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES, + }; + + VkImageFormatProperties2 format_properties = { + .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2, + .pNext = &external_format_properties, + }; + + ret = vk->vkGetPhysicalDeviceImageFormatProperties2(vk->physical_device, &format_info, &format_properties); + if (ret != VK_SUCCESS) { + VK_ERROR(vk, "vkGetPhysicalDeviceImageFormatProperties2: %s", vk_result_string(ret)); + // Nothing to cleanup + return ret; + } + + VkExternalMemoryFeatureFlags features = + external_format_properties.externalMemoryProperties.externalMemoryFeatures; + + if ((features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT) == 0) { + VK_ERROR(vk, "External memory handle is not importable (has features: %d)", features); + return VK_ERROR_INITIALIZATION_FAILED; + } + VkImageCreateInfo vk_info = { .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .pNext = &external_memory_image_create_info, @@ -342,6 +392,10 @@ vk_create_image_from_native(struct vk_bundle *vk, .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, }; + if (0 != (info->create & XRT_SWAPCHAIN_CREATE_PROTECTED_CONTENT)) { + vk_info.flags |= VK_IMAGE_CREATE_PROTECTED_BIT; + } + ret = vk->vkCreateImage(vk->device, &vk_info, NULL, &image); if (ret != VK_SUCCESS) { VK_ERROR(vk, "vkCreateImage: %s", vk_result_string(ret)); @@ -390,6 +444,68 @@ vk_create_image_from_native(struct vk_bundle *vk, return ret; } +VkResult +vk_create_fence_sync_from_native(struct vk_bundle *vk, xrt_graphics_sync_handle_t native, VkFence *out_fence) +{ + VkFence fence = VK_NULL_HANDLE; + VkResult ret; + + VkFenceCreateInfo create_info = { + .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, + .flags = VK_FENCE_CREATE_SIGNALED_BIT, + }; + + ret = vk->vkCreateFence(vk->device, &create_info, NULL, &fence); + if (ret != VK_SUCCESS) { + VK_ERROR(vk, "vkCreateFence: %s", vk_result_string(ret)); + return ret; + } + +#ifdef XRT_GRAPHICS_SYNC_HANDLE_IS_FD + // This is what is used on Linux Mesa when importing fences from OpenGL. + VkExternalFenceHandleTypeFlagBits handleType = VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT; + + VkImportFenceFdInfoKHR import_info = { + .sType = VK_STRUCTURE_TYPE_IMPORT_FENCE_FD_INFO_KHR, + .fence = fence, + .handleType = handleType, + .fd = native, + }; + + ret = vk->vkImportFenceFdKHR(vk->device, &import_info); + if (ret != VK_SUCCESS) { + vk->vkDestroyFence(vk->device, fence, NULL); + VK_ERROR(vk, "vkImportFenceFdKHR: %s", vk_result_string(ret)); + return ret; + } +#elif defined(XRT_GRAPHICS_SYNC_HANDLE_IS_WIN32_HANDLE) + //! @todo make sure this is the right one + VkExternalFenceHandleTypeFlagBits handleType = VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_WIN32_BIT; + VkImportFenceWin32HandleInfoKHR import_info = { + .sType = VK_STRUCTURE_TYPE_IMPORT_FENCE_WIN32_HANDLE_INFO_KHR, + .pNext = NULL, + .fence = fence, + .flags = 0, /** @todo do we want the temporary flag? */ + .handleType = handleType, + .handle = native, + .name = NULL, /* not importing by name */ + }; + + ret = vk->vkImportFenceWin32HandleKHR(vk->device, &import_info); + if (ret != VK_SUCCESS) { + vk->vkDestroyFence(vk->device, fence, NULL); + VK_ERROR(vk, "vkImportFenceFdKHR: %s", vk_result_string(ret)); + return ret; + } +#else +#error "Need port to import fence sync handles" +#endif + + *out_fence = fence; + + return VK_SUCCESS; +} + VkResult vk_create_semaphore_from_native(struct vk_bundle *vk, xrt_graphics_sync_handle_t native, VkSemaphore *out_sem) { @@ -710,6 +826,7 @@ vk_get_instance_functions(struct vk_bundle *vk) vk->vkEnumeratePhysicalDevices = GET_INS_PROC(vk, vkEnumeratePhysicalDevices); vk->vkGetPhysicalDeviceProperties = GET_INS_PROC(vk, vkGetPhysicalDeviceProperties); vk->vkGetPhysicalDeviceProperties2 = GET_INS_PROC(vk, vkGetPhysicalDeviceProperties2); + vk->vkGetPhysicalDeviceFeatures2 = GET_INS_PROC(vk, vkGetPhysicalDeviceFeatures2); vk->vkGetPhysicalDeviceMemoryProperties = GET_INS_PROC(vk, vkGetPhysicalDeviceMemoryProperties); vk->vkGetPhysicalDeviceQueueFamilyProperties = GET_INS_PROC(vk, vkGetPhysicalDeviceQueueFamilyProperties); vk->vkCreateDebugReportCallbackEXT = GET_INS_PROC(vk, vkCreateDebugReportCallbackEXT); @@ -721,6 +838,15 @@ vk_get_instance_functions(struct vk_bundle *vk) vk->vkGetPhysicalDeviceSurfaceSupportKHR = GET_INS_PROC(vk, vkGetPhysicalDeviceSurfaceSupportKHR); vk->vkGetPhysicalDeviceFormatProperties = GET_INS_PROC(vk, vkGetPhysicalDeviceFormatProperties); vk->vkEnumerateDeviceExtensionProperties = GET_INS_PROC(vk, vkEnumerateDeviceExtensionProperties); + vk->vkGetPhysicalDeviceImageFormatProperties2 = GET_INS_PROC(vk, vkGetPhysicalDeviceImageFormatProperties2); +#ifdef VK_USE_PLATFORM_DISPLAY_KHR + vk->vkCreateDisplayPlaneSurfaceKHR = GET_INS_PROC(vk, vkCreateDisplayPlaneSurfaceKHR); + vk->vkGetDisplayPlaneCapabilitiesKHR = GET_INS_PROC(vk, vkGetDisplayPlaneCapabilitiesKHR); + vk->vkGetPhysicalDeviceDisplayPropertiesKHR = GET_INS_PROC(vk, vkGetPhysicalDeviceDisplayPropertiesKHR); + vk->vkGetPhysicalDeviceDisplayPlanePropertiesKHR = GET_INS_PROC(vk, vkGetPhysicalDeviceDisplayPlanePropertiesKHR); + vk->vkGetDisplayModePropertiesKHR = GET_INS_PROC(vk, vkGetDisplayModePropertiesKHR); + vk->vkReleaseDisplayEXT = GET_INS_PROC(vk, vkReleaseDisplayEXT); +#endif #ifdef VK_USE_PLATFORM_XCB_KHR vk->vkCreateXcbSurfaceKHR = GET_INS_PROC(vk, vkCreateXcbSurfaceKHR); @@ -728,17 +854,16 @@ vk_get_instance_functions(struct vk_bundle *vk) #ifdef VK_USE_PLATFORM_WAYLAND_KHR vk->vkCreateWaylandSurfaceKHR = GET_INS_PROC(vk, vkCreateWaylandSurfaceKHR); + +#ifdef VK_EXT_acquire_drm_display + vk->vkAcquireDrmDisplayEXT = GET_INS_PROC(vk, vkAcquireDrmDisplayEXT); + vk->vkGetDrmDisplayEXT = GET_INS_PROC(vk, vkGetDrmDisplayEXT); +#endif #endif #ifdef VK_USE_PLATFORM_XLIB_XRANDR_EXT - vk->vkCreateDisplayPlaneSurfaceKHR = GET_INS_PROC(vk, vkCreateDisplayPlaneSurfaceKHR); - vk->vkGetDisplayPlaneCapabilitiesKHR = GET_INS_PROC(vk, vkGetDisplayPlaneCapabilitiesKHR); - vk->vkGetPhysicalDeviceDisplayPropertiesKHR = GET_INS_PROC(vk, vkGetPhysicalDeviceDisplayPropertiesKHR); - vk->vkGetPhysicalDeviceDisplayPlanePropertiesKHR = GET_INS_PROC(vk, vkGetPhysicalDeviceDisplayPlanePropertiesKHR); - vk->vkGetDisplayModePropertiesKHR = GET_INS_PROC(vk, vkGetDisplayModePropertiesKHR); - vk->vkAcquireXlibDisplayEXT = GET_INS_PROC(vk, vkAcquireXlibDisplayEXT); - vk->vkReleaseDisplayEXT = GET_INS_PROC(vk, vkReleaseDisplayEXT); vk->vkGetRandROutputDisplayEXT = GET_INS_PROC(vk, vkGetRandROutputDisplayEXT); + vk->vkAcquireXlibDisplayEXT = GET_INS_PROC(vk, vkAcquireXlibDisplayEXT); #endif #ifdef VK_USE_PLATFORM_ANDROID_KHR @@ -780,6 +905,8 @@ vk_get_device_functions(struct vk_bundle *vk) vk->vkFlushMappedMemoryRanges = GET_DEV_PROC(vk, vkFlushMappedMemoryRanges); vk->vkCreateImage = GET_DEV_PROC(vk, vkCreateImage); vk->vkGetImageMemoryRequirements = GET_DEV_PROC(vk, vkGetImageMemoryRequirements); + // because we use Vulkan API Version 1.0.x, we can only get the KHR version of this function + vk->vkGetImageMemoryRequirements2 = GET_DEV_PROC(vk, vkGetImageMemoryRequirements2KHR); vk->vkBindImageMemory = GET_DEV_PROC(vk, vkBindImageMemory); vk->vkDestroyImage = GET_DEV_PROC(vk, vkDestroyImage); vk->vkCreateImageView = GET_DEV_PROC(vk, vkCreateImageView); @@ -804,6 +931,9 @@ vk_get_device_functions(struct vk_bundle *vk) vk->vkCmdBindIndexBuffer = GET_DEV_PROC(vk, vkCmdBindIndexBuffer); vk->vkCmdDraw = GET_DEV_PROC(vk, vkCmdDraw); vk->vkCmdDrawIndexed = GET_DEV_PROC(vk, vkCmdDrawIndexed); + vk->vkCmdDispatch = GET_DEV_PROC(vk, vkCmdDispatch); + vk->vkCmdCopyBufferToImage = GET_DEV_PROC(vk, vkCmdCopyBufferToImage); + vk->vkCmdCopyImageToBuffer = GET_DEV_PROC(vk, vkCmdCopyImageToBuffer); vk->vkEndCommandBuffer = GET_DEV_PROC(vk, vkEndCommandBuffer); vk->vkFreeCommandBuffers = GET_DEV_PROC(vk, vkFreeCommandBuffers); vk->vkCreateRenderPass = GET_DEV_PROC(vk, vkCreateRenderPass); @@ -812,10 +942,12 @@ vk_get_device_functions(struct vk_bundle *vk) vk->vkDestroyFramebuffer = GET_DEV_PROC(vk, vkDestroyFramebuffer); vk->vkCreatePipelineCache = GET_DEV_PROC(vk, vkCreatePipelineCache); vk->vkDestroyPipelineCache = GET_DEV_PROC(vk, vkDestroyPipelineCache); + vk->vkResetDescriptorPool = GET_DEV_PROC(vk, vkResetDescriptorPool); vk->vkCreateDescriptorPool = GET_DEV_PROC(vk, vkCreateDescriptorPool); vk->vkDestroyDescriptorPool = GET_DEV_PROC(vk, vkDestroyDescriptorPool); vk->vkAllocateDescriptorSets = GET_DEV_PROC(vk, vkAllocateDescriptorSets); vk->vkFreeDescriptorSets = GET_DEV_PROC(vk, vkFreeDescriptorSets); + vk->vkCreateComputePipelines = GET_DEV_PROC(vk, vkCreateComputePipelines); vk->vkCreateGraphicsPipelines = GET_DEV_PROC(vk, vkCreateGraphicsPipelines); vk->vkDestroyPipeline = GET_DEV_PROC(vk, vkDestroyPipeline); vk->vkCreatePipelineLayout = GET_DEV_PROC(vk, vkCreatePipelineLayout); @@ -830,6 +962,7 @@ vk_get_device_functions(struct vk_bundle *vk) vk->vkDestroySemaphore = GET_DEV_PROC(vk, vkDestroySemaphore); vk->vkCreateFence = GET_DEV_PROC(vk, vkCreateFence); vk->vkWaitForFences = GET_DEV_PROC(vk, vkWaitForFences); + vk->vkGetFenceStatus = GET_DEV_PROC(vk, vkGetFenceStatus); vk->vkDestroyFence = GET_DEV_PROC(vk, vkDestroyFence); vk->vkResetFences = GET_DEV_PROC(vk, vkResetFences); vk->vkCreateSwapchainKHR = GET_DEV_PROC(vk, vkCreateSwapchainKHR); @@ -840,10 +973,17 @@ vk_get_device_functions(struct vk_bundle *vk) #ifdef VK_USE_PLATFORM_WIN32_KHR vk->vkImportSemaphoreWin32HandleKHR = GET_DEV_PROC(vk, vkImportSemaphoreWin32HandleKHR); + vk->vkImportFenceWin32HandleKHR = GET_DEV_PROC(vk, vkImportFenceWin32HandleKHR); #else vk->vkImportSemaphoreFdKHR = GET_DEV_PROC(vk, vkImportSemaphoreFdKHR); vk->vkGetSemaphoreFdKHR = GET_DEV_PROC(vk, vkGetSemaphoreFdKHR); + + vk->vkImportFenceFdKHR = GET_DEV_PROC(vk, vkImportFenceFdKHR); + vk->vkGetFenceFdKHR = GET_DEV_PROC(vk, vkGetFenceFdKHR); #endif + + vk->vkGetPastPresentationTimingGOOGLE = GET_DEV_PROC(vk, vkGetPastPresentationTimingGOOGLE); + // clang-format on return VK_SUCCESS; @@ -898,10 +1038,8 @@ vk_select_physical_device(struct vk_bundle *vk, int forced_index) uint32_t gpu_index = 0; if (forced_index > -1) { if ((uint32_t)forced_index + 1 > gpu_count) { - VK_ERROR(vk, - "Attempted to force GPU index %d, but only %d " - "GPUs are available", - forced_index, gpu_count); + VK_ERROR(vk, "Attempted to force GPU index %d, but only %d GPUs are available", forced_index, + gpu_count); return VK_ERROR_DEVICE_LOST; } gpu_index = forced_index; @@ -934,6 +1072,12 @@ vk_select_physical_device(struct vk_bundle *vk, int forced_index) snprintf(title, 20, "Selected GPU: %d\n", gpu_index); vk_print_device_info_debug(vk, &pdp, gpu_index, title); + char *tegra_substr = strstr(pdp.deviceName, "Tegra"); + if (tegra_substr) { + vk->is_tegra = true; + VK_DEBUG(vk, "Detected Tegra, using Tegra specific workarounds!"); + } + // Fill out the device memory props as well. vk->vkGetPhysicalDeviceMemoryProperties(vk->physical_device, &vk->device_memory_props); @@ -979,16 +1123,55 @@ err_free: return VK_ERROR_INITIALIZATION_FAILED; } +static VkResult +vk_find_compute_only_queue(struct vk_bundle *vk, uint32_t *out_compute_queue) +{ + /* Find the first graphics queue */ + uint32_t num_queues = 0; + uint32_t i = 0; + vk->vkGetPhysicalDeviceQueueFamilyProperties(vk->physical_device, &num_queues, NULL); + + VkQueueFamilyProperties *queue_family_props = U_TYPED_ARRAY_CALLOC(VkQueueFamilyProperties, num_queues); + + vk->vkGetPhysicalDeviceQueueFamilyProperties(vk->physical_device, &num_queues, queue_family_props); + + if (num_queues == 0) { + VK_DEBUG(vk, "Failed to get queue properties"); + goto err_free; + } + + for (i = 0; i < num_queues; i++) { + if (queue_family_props[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) { + continue; + } + + if (queue_family_props[i].queueFlags & VK_QUEUE_COMPUTE_BIT) { + break; + } + } + + if (i >= num_queues) { + VK_DEBUG(vk, "No compute only queue found"); + goto err_free; + } + + *out_compute_queue = i; + + free(queue_family_props); + + return VK_SUCCESS; + +err_free: + free(queue_family_props); + + return VK_ERROR_INITIALIZATION_FAILED; +} + static bool vk_check_extension(struct vk_bundle *vk, VkExtensionProperties *props, uint32_t num_props, const char *ext) { for (uint32_t i = 0; i < num_props; i++) { if (strcmp(props[i].extensionName, ext) == 0) { - - if (strcmp(ext, VK_GOOGLE_DISPLAY_TIMING_EXTENSION_NAME) == 0) { - vk->has_GOOGLE_display_timing = true; - } - return true; } } @@ -996,53 +1179,81 @@ vk_check_extension(struct vk_bundle *vk, VkExtensionProperties *props, uint32_t return false; } -static bool +static void +fill_in_has_extensions(struct vk_bundle *vk, const char **device_extensions, uint32_t num_device_extensions) +{ + // Reset before filling out. + vk->has_GOOGLE_display_timing = false; + vk->has_EXT_global_priority = false; + vk->has_VK_EXT_robustness2 = false; + + for (uint32_t i = 0; i < num_device_extensions; i++) { + const char *ext = device_extensions[i]; + + if (strcmp(ext, VK_GOOGLE_DISPLAY_TIMING_EXTENSION_NAME) == 0) { + vk->has_GOOGLE_display_timing = true; + } + if (strcmp(ext, VK_EXT_GLOBAL_PRIORITY_EXTENSION_NAME) == 0) { + vk->has_EXT_global_priority = true; + } +#ifdef VK_EXT_robustness2 + if (strcmp(ext, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME) == 0) { + vk->has_VK_EXT_robustness2 = true; + } +#endif + } +} + +static VkResult vk_get_device_ext_props(struct vk_bundle *vk, VkPhysicalDevice physical_device, - VkExtensionProperties **props, - uint32_t *num_props) + VkExtensionProperties **out_props, + uint32_t *out_num_props) { - VkResult res = vk->vkEnumerateDeviceExtensionProperties(physical_device, NULL, num_props, NULL); + uint32_t num_props = 0; + VkResult res = vk->vkEnumerateDeviceExtensionProperties(physical_device, NULL, &num_props, NULL); vk_check_error("vkEnumerateDeviceExtensionProperties", res, false); - *props = U_TYPED_ARRAY_CALLOC(VkExtensionProperties, *num_props); + VkExtensionProperties *props = U_TYPED_ARRAY_CALLOC(VkExtensionProperties, num_props); - res = vk->vkEnumerateDeviceExtensionProperties(physical_device, NULL, num_props, *props); + res = vk->vkEnumerateDeviceExtensionProperties(physical_device, NULL, &num_props, props); vk_check_error_with_free("vkEnumerateDeviceExtensionProperties", res, false, props); - return true; + // Check above returns on failure. + *out_props = props; + *out_num_props = num_props; + + return VK_SUCCESS; } static bool vk_build_device_extensions(struct vk_bundle *vk, VkPhysicalDevice physical_device, const char *const *required_device_extensions, - size_t num_required_device_extensions, + uint32_t num_required_device_extensions, const char *const *optional_device_extensions, - size_t num_optional_device_extensions, + uint32_t num_optional_device_extensions, const char ***out_device_extensions, - size_t *out_num_device_extensions) + uint32_t *out_num_device_extensions) { - VkExtensionProperties *props; - uint32_t num_props; - if (!vk_get_device_ext_props(vk, physical_device, &props, &num_props)) { + VkExtensionProperties *props = NULL; + uint32_t num_props = 0; + if (vk_get_device_ext_props(vk, physical_device, &props, &num_props) != VK_SUCCESS) { return false; } - int max_exts = num_required_device_extensions + num_optional_device_extensions; + uint32_t max_exts = num_required_device_extensions + num_optional_device_extensions; const char **device_extensions = U_TYPED_ARRAY_CALLOC(const char *, max_exts); for (uint32_t i = 0; i < num_required_device_extensions; i++) { const char *ext = required_device_extensions[i]; if (!vk_check_extension(vk, props, num_props, ext)) { - U_LOG_E( - "VkPhysicalDevice does not support required " - "extension %s", - ext); + U_LOG_E("VkPhysicalDevice does not support required extension %s", ext); free(props); return false; } + U_LOG_T("Using required device ext %s", ext); device_extensions[i] = ext; } @@ -1050,13 +1261,17 @@ vk_build_device_extensions(struct vk_bundle *vk, for (uint32_t i = 0; i < num_optional_device_extensions; i++) { const char *ext = optional_device_extensions[i]; if (vk_check_extension(vk, props, num_props, ext)) { - U_LOG_D("Using optional ext %s", ext); + U_LOG_D("Using optional device ext %s", ext); device_extensions[num_device_extensions++] = ext; } else { + U_LOG_T("NOT using optional device ext %s", ext); continue; } } + // Fill this out here. + fill_in_has_extensions(vk, device_extensions, num_device_extensions); + free(props); *out_device_extensions = device_extensions; @@ -1065,13 +1280,77 @@ vk_build_device_extensions(struct vk_bundle *vk, return true; } +static void +filter_device_features(struct vk_bundle *vk, + VkPhysicalDevice physical_device, + const struct vk_device_features *optional_device_features, + struct vk_device_features *device_features) +{ + // If no features are requested, then noop. + if (optional_device_features == NULL) { + return; + } + + /* + * The structs + */ + +#ifdef VK_EXT_robustness2 + VkPhysicalDeviceRobustness2FeaturesEXT robust_info = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT, + .pNext = NULL, + }; +#endif + + VkPhysicalDeviceFeatures2 physical_device_features = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, + .pNext = NULL, + }; + +#ifdef VK_EXT_robustness2 + if (vk->has_VK_EXT_robustness2) { + physical_device_features.pNext = physical_device_features.pNext; + physical_device_features.pNext = (void *)&robust_info; + } +#endif + + vk->vkGetPhysicalDeviceFeatures2( // + physical_device, // physicalDevice + &physical_device_features); // pFeatures + + + /* + * Collect and transfer. + */ + +#define CHECK(feature, DEV_FEATURE) device_features->feature = optional_device_features->feature && (DEV_FEATURE) + +#ifdef VK_EXT_robustness2 + CHECK(null_descriptor, robust_info.nullDescriptor); +#endif + CHECK(shader_storage_image_write_without_format, + physical_device_features.features.shaderStorageImageWriteWithoutFormat); + +#undef CHECK + + + VK_DEBUG(vk, + "Features:" + "\n\tnull_descriptor: %i" + "\n\tshader_storage_image_write_without_format: %i", + device_features->null_descriptor, device_features->shader_storage_image_write_without_format); +} + VkResult vk_create_device(struct vk_bundle *vk, int forced_index, + bool only_compute, + VkQueueGlobalPriorityEXT global_priority, const char *const *required_device_extensions, size_t num_required_device_extensions, const char *const *optional_device_extensions, - size_t num_optional_device_extensions) + size_t num_optional_device_extensions, + const struct vk_device_features *optional_device_features) { VkResult ret; @@ -1081,42 +1360,99 @@ vk_create_device(struct vk_bundle *vk, } const char **device_extensions; - size_t num_device_extensions; + uint32_t num_device_extensions; if (!vk_build_device_extensions(vk, vk->physical_device, required_device_extensions, num_required_device_extensions, optional_device_extensions, num_optional_device_extensions, &device_extensions, &num_device_extensions)) { return VK_ERROR_EXTENSION_NOT_PRESENT; } - VkPhysicalDeviceFeatures *enabled_features = NULL; + + /* + * Features + */ + + struct vk_device_features device_features = {0}; + filter_device_features(vk, vk->physical_device, optional_device_features, &device_features); + + + /* + * Queue + */ + + if (only_compute) { + ret = vk_find_compute_only_queue(vk, &vk->queue_family_index); + } else { + ret = vk_find_graphics_queue(vk, &vk->queue_family_index); + } + + if (ret != VK_SUCCESS) { + return ret; + } + + VkDeviceQueueGlobalPriorityCreateInfoEXT priority_info = { + .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_EXT, + .pNext = NULL, + .globalPriority = global_priority, + }; float queue_priority = 0.0f; VkDeviceQueueCreateInfo queue_create_info = { .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, + .pNext = NULL, .queueCount = 1, + .queueFamilyIndex = vk->queue_family_index, .pQueuePriorities = &queue_priority, }; - ret = vk_find_graphics_queue(vk, &queue_create_info.queueFamilyIndex); - if (ret != VK_SUCCESS) { - return ret; + if (vk->has_EXT_global_priority) { + priority_info.pNext = queue_create_info.pNext; + queue_create_info.pNext = (void *)&priority_info; } + + /* + * Device + */ + +#ifdef VK_EXT_robustness2 + VkPhysicalDeviceRobustness2FeaturesEXT robust_info = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT, + .pNext = NULL, + .nullDescriptor = device_features.null_descriptor, + }; +#endif + + VkPhysicalDeviceFeatures enabled_features = { + .shaderStorageImageWriteWithoutFormat = device_features.shader_storage_image_write_without_format, + }; + VkDeviceCreateInfo device_create_info = { .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, .queueCreateInfoCount = 1, .pQueueCreateInfos = &queue_create_info, .enabledExtensionCount = num_device_extensions, .ppEnabledExtensionNames = device_extensions, - .pEnabledFeatures = enabled_features, + .pEnabledFeatures = &enabled_features, }; +#ifdef VK_EXT_robustness2 + if (vk->has_VK_EXT_robustness2) { + // This struct is a in/out struct, while device_create_info has a const pNext. + robust_info.pNext = (void *)device_create_info.pNext; + device_create_info.pNext = (void *)&robust_info; + } +#endif + ret = vk->vkCreateDevice(vk->physical_device, &device_create_info, NULL, &vk->device); free(device_extensions); if (ret != VK_SUCCESS) { - VK_DEBUG(vk, "vkCreateDevice: %s", vk_result_string(ret)); + VK_DEBUG(vk, "vkCreateDevice: %s (%d)", vk_result_string(ret), ret); + if (ret == VK_ERROR_NOT_PERMITTED_EXT) { + VK_DEBUG(vk, "Is CAP_SYS_NICE set? Try: sudo setcap cap_sys_nice+ep monado-service"); + } return ret; } @@ -1135,6 +1471,26 @@ err_destroy: return ret; } +VkResult +vk_init_mutex(struct vk_bundle *vk) +{ + if (os_mutex_init(&vk->cmd_pool_mutex) < 0) { + return VK_ERROR_INITIALIZATION_FAILED; + } + if (os_mutex_init(&vk->queue_mutex) < 0) { + return VK_ERROR_INITIALIZATION_FAILED; + } + return VK_SUCCESS; +} + +VkResult +vk_deinit_mutex(struct vk_bundle *vk) +{ + os_mutex_destroy(&vk->cmd_pool_mutex); + os_mutex_destroy(&vk->queue_mutex); + return VK_SUCCESS; +} + VkResult vk_init_from_given(struct vk_bundle *vk, PFN_vkGetInstanceProcAddr vkGetInstanceProcAddr, @@ -1183,6 +1539,7 @@ vk_init_from_given(struct vk_bundle *vk, goto err_memset; } + return VK_SUCCESS; err_memset: @@ -1245,11 +1602,9 @@ check_feature(VkFormat format, VkFormatFeatureFlags flag) { if ((format_features & flag) == 0) { - U_LOG_E( - "vk_swapchain_usage_flags: %s requested but %s not " - "supported for format %s (%08x) (%08x)", - xrt_swapchain_usage_string(usage), vk_format_feature_string(flag), vk_color_format_string(format), - format_features, flag); + U_LOG_E("vk_swapchain_usage_flags: %s requested but %s not supported for format %s (%08x) (%08x)", + xrt_swapchain_usage_string(usage), vk_format_feature_string(flag), + vk_color_format_string(format), format_features, flag); return false; } return true; @@ -1370,8 +1725,10 @@ vk_buffer_init(struct vk_bundle *vk, VkMemoryRequirements requirements; vk->vkGetBufferMemoryRequirements(vk->device, *out_buffer, &requirements); - VkMemoryAllocateInfo alloc_info = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, - .allocationSize = requirements.size}; + VkMemoryAllocateInfo alloc_info = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .allocationSize = requirements.size, + }; if (!vk_get_memory_type(vk, requirements.memoryTypeBits, properties, &alloc_info.memoryTypeIndex)) { VK_ERROR(vk, "Failed to find matching memoryTypeIndex for buffer"); diff --git a/src/xrt/auxiliary/vk/vk_helpers.h b/src/xrt/auxiliary/vk/vk_helpers.h index e3e930bf2..662dbb660 100644 --- a/src/xrt/auxiliary/vk/vk_helpers.h +++ b/src/xrt/auxiliary/vk/vk_helpers.h @@ -49,6 +49,10 @@ struct vk_bundle struct os_mutex queue_mutex; bool has_GOOGLE_display_timing; + bool has_EXT_global_priority; + bool has_VK_EXT_robustness2; + + bool is_tegra; VkDebugReportCallbackEXT debug_report_cb; @@ -71,6 +75,9 @@ struct vk_bundle PFN_vkEnumeratePhysicalDevices vkEnumeratePhysicalDevices; PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR; PFN_vkEnumerateDeviceExtensionProperties vkEnumerateDeviceExtensionProperties; +#ifdef VK_USE_PLATFORM_DISPLAY_KHR + PFN_vkCreateDisplayPlaneSurfaceKHR vkCreateDisplayPlaneSurfaceKHR; +#endif #ifdef VK_USE_PLATFORM_XCB_KHR PFN_vkCreateXcbSurfaceKHR vkCreateXcbSurfaceKHR; @@ -78,19 +85,14 @@ struct vk_bundle #ifdef VK_USE_PLATFORM_WAYLAND_KHR PFN_vkCreateWaylandSurfaceKHR vkCreateWaylandSurfaceKHR; +#ifdef VK_EXT_acquire_drm_display + PFN_vkAcquireDrmDisplayEXT vkAcquireDrmDisplayEXT; + PFN_vkGetDrmDisplayEXT vkGetDrmDisplayEXT; +#endif #endif #ifdef VK_USE_PLATFORM_XLIB_XRANDR_EXT - PFN_vkCreateDisplayPlaneSurfaceKHR vkCreateDisplayPlaneSurfaceKHR; - PFN_vkGetDisplayPlaneCapabilitiesKHR vkGetDisplayPlaneCapabilitiesKHR; - - // This doesn't strictly require VK_USE_PLATFORM_XLIB_XRANDR_EXT, - // but it's only used in the NVIDIA X direct mode path that does require it. - PFN_vkGetPhysicalDeviceDisplayPropertiesKHR vkGetPhysicalDeviceDisplayPropertiesKHR; - PFN_vkGetPhysicalDeviceDisplayPlanePropertiesKHR vkGetPhysicalDeviceDisplayPlanePropertiesKHR; - PFN_vkGetDisplayModePropertiesKHR vkGetDisplayModePropertiesKHR; PFN_vkAcquireXlibDisplayEXT vkAcquireXlibDisplayEXT; - PFN_vkReleaseDisplayEXT vkReleaseDisplayEXT; PFN_vkGetRandROutputDisplayEXT vkGetRandROutputDisplayEXT; #endif @@ -107,6 +109,7 @@ struct vk_bundle PFN_vkGetPhysicalDeviceQueueFamilyProperties vkGetPhysicalDeviceQueueFamilyProperties; PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties; PFN_vkGetPhysicalDeviceProperties2 vkGetPhysicalDeviceProperties2; + PFN_vkGetPhysicalDeviceFeatures2 vkGetPhysicalDeviceFeatures2; PFN_vkGetPhysicalDeviceSurfaceCapabilitiesKHR vkGetPhysicalDeviceSurfaceCapabilitiesKHR; PFN_vkGetPhysicalDeviceSurfaceFormatsKHR vkGetPhysicalDeviceSurfaceFormatsKHR; @@ -115,6 +118,15 @@ struct vk_bundle PFN_vkGetPhysicalDeviceFormatProperties vkGetPhysicalDeviceFormatProperties; + PFN_vkGetPhysicalDeviceImageFormatProperties2 vkGetPhysicalDeviceImageFormatProperties2; + +#ifdef VK_USE_PLATFORM_DISPLAY_KHR + PFN_vkGetDisplayModePropertiesKHR vkGetDisplayModePropertiesKHR; + PFN_vkGetPhysicalDeviceDisplayPropertiesKHR vkGetPhysicalDeviceDisplayPropertiesKHR; + PFN_vkGetPhysicalDeviceDisplayPlanePropertiesKHR vkGetPhysicalDeviceDisplayPlanePropertiesKHR; + PFN_vkGetDisplayPlaneCapabilitiesKHR vkGetDisplayPlaneCapabilitiesKHR; + PFN_vkReleaseDisplayEXT vkReleaseDisplayEXT; +#endif // Device functions. PFN_vkGetDeviceProcAddr vkGetDeviceProcAddr; @@ -144,6 +156,7 @@ struct vk_bundle PFN_vkCreateImage vkCreateImage; PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements; + PFN_vkGetImageMemoryRequirements2 vkGetImageMemoryRequirements2; PFN_vkBindImageMemory vkBindImageMemory; PFN_vkDestroyImage vkDestroyImage; PFN_vkCreateImageView vkCreateImageView; @@ -171,6 +184,9 @@ struct vk_bundle PFN_vkCmdBindIndexBuffer vkCmdBindIndexBuffer; PFN_vkCmdDraw vkCmdDraw; PFN_vkCmdDrawIndexed vkCmdDrawIndexed; + PFN_vkCmdDispatch vkCmdDispatch; + PFN_vkCmdCopyBufferToImage vkCmdCopyBufferToImage; + PFN_vkCmdCopyImageToBuffer vkCmdCopyImageToBuffer; PFN_vkEndCommandBuffer vkEndCommandBuffer; PFN_vkFreeCommandBuffers vkFreeCommandBuffers; @@ -180,10 +196,12 @@ struct vk_bundle PFN_vkDestroyFramebuffer vkDestroyFramebuffer; PFN_vkCreatePipelineCache vkCreatePipelineCache; PFN_vkDestroyPipelineCache vkDestroyPipelineCache; + PFN_vkResetDescriptorPool vkResetDescriptorPool; PFN_vkCreateDescriptorPool vkCreateDescriptorPool; PFN_vkDestroyDescriptorPool vkDestroyDescriptorPool; PFN_vkAllocateDescriptorSets vkAllocateDescriptorSets; PFN_vkFreeDescriptorSets vkFreeDescriptorSets; + PFN_vkCreateComputePipelines vkCreateComputePipelines; PFN_vkCreateGraphicsPipelines vkCreateGraphicsPipelines; PFN_vkDestroyPipeline vkDestroyPipeline; PFN_vkCreatePipelineLayout vkCreatePipelineLayout; @@ -201,6 +219,7 @@ struct vk_bundle PFN_vkCreateFence vkCreateFence; PFN_vkWaitForFences vkWaitForFences; + PFN_vkGetFenceStatus vkGetFenceStatus; PFN_vkDestroyFence vkDestroyFence; PFN_vkResetFences vkResetFences; @@ -212,10 +231,16 @@ struct vk_bundle #ifdef VK_USE_PLATFORM_WIN32_KHR PFN_vkImportSemaphoreWin32HandleKHR vkImportSemaphoreWin32HandleKHR; + PFN_vkImportFenceWin32HandleKHR vkImportFenceWin32HandleKHR; #else PFN_vkImportSemaphoreFdKHR vkImportSemaphoreFdKHR; PFN_vkGetSemaphoreFdKHR vkGetSemaphoreFdKHR; + + PFN_vkImportFenceFdKHR vkImportFenceFdKHR; + PFN_vkGetFenceFdKHR vkGetFenceFdKHR; #endif + + PFN_vkGetPastPresentationTimingGOOGLE vkGetPastPresentationTimingGOOGLE; // clang-format on }; @@ -262,18 +287,59 @@ vk_color_space_string(VkColorSpaceKHR code); #define VK_WARN(d, ...) U_LOG_IFL_W(d->ll, __VA_ARGS__) #define VK_ERROR(d, ...) U_LOG_IFL_E(d->ll, __VA_ARGS__) +/*! + * @brief Check a Vulkan VkResult, writing an error to the log and returning true if not VK_SUCCESS + * + * @param fun a string literal with the name of the Vulkan function, for logging purposes. + * @param res a VkResult from that function. + * @param file a string literal with the source code filename, such as from __FILE__ + * @param line a source code line number, such as from __LINE__ + * + * @see vk_check_error, vk_check_error_with_free which wrap this for easier usage. + * + * @ingroup aux_vk + */ bool vk_has_error(VkResult res, const char *fun, const char *file, int line); +/*! + * @def vk_check_error + * @brief Perform checking of a Vulkan result, returning in case it is not VK_SUCCESS. + * + * @param fun A string literal with the name of the Vulkan function, for logging purposes. + * @param res a VkResult from that function. + * @param ret value to return, if any, upon error + * + * @see vk_has_error which is wrapped by this macro + * + * @ingroup aux_vk + */ #define vk_check_error(fun, res, ret) \ - if (vk_has_error(res, fun, __FILE__, __LINE__)) \ - return ret + do { \ + if (vk_has_error(res, fun, __FILE__, __LINE__)) \ + return ret; \ + } while (0) +/*! + * @def vk_check_error_with_free + * @brief Perform checking of a Vulkan result, freeing an allocation and returning in case it is not VK_SUCCESS. + * + * @param fun A string literal with the name of the Vulkan function, for logging purposes. + * @param res a VkResult from that function. + * @param ret value to return, if any, upon error + * @param to_free expression to pass to `free()` upon error + * + * @see vk_has_error which is wrapped by this macro + * + * @ingroup aux_vk + */ #define vk_check_error_with_free(fun, res, ret, to_free) \ - if (vk_has_error(res, fun, __FILE__, __LINE__)) { \ - free(to_free); \ - return ret; \ - } + do { \ + if (vk_has_error(res, fun, __FILE__, __LINE__)) { \ + free(to_free); \ + return ret; \ + } \ + } while (0) /*! * @ingroup aux_vk @@ -287,22 +353,53 @@ vk_get_loader_functions(struct vk_bundle *vk, PFN_vkGetInstanceProcAddr g); VkResult vk_get_instance_functions(struct vk_bundle *vk); +/*! + * @brief Initialize mutexes in the @ref vk_bundle. + * + * Not required for all uses, but a precondition for some. + * + * @ingroup aux_vk + */ +VkResult +vk_init_mutex(struct vk_bundle *vk); + +/*! + * @brief De-initialize mutexes in the @ref vk_bundle. + * @ingroup aux_vk + */ +VkResult +vk_deinit_mutex(struct vk_bundle *vk); + /*! * @ingroup aux_vk */ VkResult vk_init_cmd_pool(struct vk_bundle *vk); +/*! + * Used to enable device features as a argument @ref vk_create_device. + * + * @ingroup aux_vk + */ +struct vk_device_features +{ + bool shader_storage_image_write_without_format; + bool null_descriptor; +}; + /*! * @ingroup aux_vk */ VkResult vk_create_device(struct vk_bundle *vk, int forced_index, + bool only_compute, + VkQueueGlobalPriorityEXT global_priorty, const char *const *required_device_extensions, size_t num_required_device_extensions, const char *const *optional_device_extensions, - size_t num_optional_device_extension); + size_t num_optional_device_extension, + const struct vk_device_features *optional_device_features); /*! * Initialize a bundle with objects given to us by client code, @@ -366,6 +463,24 @@ vk_alloc_and_bind_image_memory(struct vk_bundle *vk, VkDeviceSize *out_size); /*! + * + * @brief Creates a Vulkan device memory and image from a native graphics buffer handle. + * + * In case of error, ownership is never transferred and the caller should close the handle themselves. + * + * In case of success, the underlying Vulkan functionality's ownership semantics apply: ownership of the @p image_native + * handle may have transferred, a reference may have been added, or the Vulkan objects may rely on the caller to keep + * the native handle alive until the Vulkan objects are destroyed. Which option applies depends on the particular native + * handle type used. + * + * See the corresponding specification texts: + * + * - Windows: + * https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/vkspec.html#VkImportMemoryWin32HandleInfoKHR + * - Linux: https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/vkspec.html#VkImportMemoryFdInfoKHR + * - Android: + * https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/vkspec.html#VkImportAndroidHardwareBufferInfoANDROID + * * @ingroup aux_vk */ VkResult @@ -376,6 +491,33 @@ vk_create_image_from_native(struct vk_bundle *vk, VkDeviceMemory *out_mem); /*! + * @brief Creates a Vulkan fence from a native graphics sync handle. + * + * In case of error, ownership is never transferred and the caller should close the handle themselves. + * + * In case of success, the underlying Vulkan functionality's ownership semantics apply: ownership of the @p native + * handle may have transferred, a reference may have been added, or the Vulkan object may rely on the caller to keep the + * native handle alive until the Vulkan object is destroyed. Which option applies depends on the particular native + * handle type used. + * + * See the corresponding Vulkan specification text: + * https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/vkspec.html#synchronization-fences-importing + * + * @ingroup aux_vk + */ +VkResult +vk_create_fence_sync_from_native(struct vk_bundle *vk, xrt_graphics_sync_handle_t native, VkFence *out_fence); + +/*! + * @brief Creates a Vulkan semaphore from a native graphics sync handle. + * + * In case of error, ownership is never transferred and the caller should close the handle themselves. + * + * In case of success, the underlying Vulkan functionality's ownership semantics apply: ownership of the @p native + * handle may have transferred, a reference may have been added, or the Vulkan object may rely on the caller to keep the + * native handle alive until the Vulkan object is destroyed. Which option applies depends on the particular native + * handle type used. + * * @ingroup aux_vk */ VkResult @@ -420,12 +562,14 @@ vk_create_view_swizzle(struct vk_bundle *vk, VkImageView *out_view); /*! + * @pre Requires successful call to vk_init_mutex * @ingroup aux_vk */ VkResult vk_init_cmd_buffer(struct vk_bundle *vk, VkCommandBuffer *out_cmd_buffer); /*! + * @pre Requires successful call to vk_init_mutex * @ingroup aux_vk */ VkResult @@ -439,6 +583,7 @@ vk_set_image_layout(struct vk_bundle *vk, VkImageSubresourceRange subresource_range); /*! + * @pre Requires successful call to vk_init_mutex * @ingroup aux_vk */ VkResult @@ -487,6 +632,10 @@ vk_buffer_destroy(struct vk_buffer *self, struct vk_bundle *vk); bool vk_update_buffer(struct vk_bundle *vk, float *buffer, size_t buffer_size, VkDeviceMemory memory); +/*! + * @pre Requires successful call to vk_init_mutex + * @ingroup aux_vk + */ VkResult vk_locked_submit(struct vk_bundle *vk, VkQueue queue, uint32_t count, const VkSubmitInfo *infos, VkFence fence); diff --git a/src/xrt/auxiliary/vk/vk_image_allocator.c b/src/xrt/auxiliary/vk/vk_image_allocator.c index d0e8c2a26..1a26e4e1b 100644 --- a/src/xrt/auxiliary/vk/vk_image_allocator.c +++ b/src/xrt/auxiliary/vk/vk_image_allocator.c @@ -155,8 +155,12 @@ create_image(struct vk_bundle *vk, const struct xrt_swapchain_create_info *info, .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO_KHR, #if defined(XRT_GRAPHICS_BUFFER_HANDLE_IS_AHARDWAREBUFFER) .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID, -#else +#elif defined(XRT_GRAPHICS_BUFFER_HANDLE_IS_WIN32_HANDLE) + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT_KHR, +#elif defined(XRT_GRAPHICS_BUFFER_HANDLE_IS_FD) .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR, +#else +#error "need port" #endif }; @@ -200,6 +204,38 @@ create_image(struct vk_bundle *vk, const struct xrt_swapchain_create_info *info, return ret; } + VkImageMemoryRequirementsInfo2 memory_requirements_info = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, + .image = image, + }; + + VkMemoryDedicatedRequirements memory_dedicated_requirements = { + .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS, + }; + VkMemoryRequirements2 memory_requirements = { + .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, + .pNext = &memory_dedicated_requirements, + }; + vk->vkGetImageMemoryRequirements2(vk->device, &memory_requirements_info, &memory_requirements); + + /* on tegra we must not use dedicated allocation when it is only preferred to avoid black textures and driver + * errors when blitting from opengl interop textures. + * + * on desktop nvidia and everywhere else we must use dedicated allocation when it is preferred to avoid fences + * timing out and driver errors "Graphics Exception on GPC 0: 3D-C MEMLAYOUT Violation." + */ + VkBool32 use_dedicated_allocation; + if (vk->is_tegra) { + use_dedicated_allocation = memory_dedicated_requirements.requiresDedicatedAllocation != VK_FALSE; + } else { + use_dedicated_allocation = (memory_dedicated_requirements.requiresDedicatedAllocation != VK_FALSE) || + (memory_dedicated_requirements.prefersDedicatedAllocation != VK_FALSE); + } + + U_LOG_D("create_image: Use dedicated allocation: %d (preferred: %d, required: %d)", use_dedicated_allocation, + memory_dedicated_requirements.prefersDedicatedAllocation, + memory_dedicated_requirements.requiresDedicatedAllocation); + /* * Create and bind the memory. */ @@ -214,7 +250,7 @@ create_image(struct vk_bundle *vk, const struct xrt_swapchain_create_info *info, VkExportMemoryAllocateInfo export_alloc_info = { .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO_KHR, - .pNext = &dedicated_memory_info, + .pNext = use_dedicated_allocation ? &dedicated_memory_info : NULL, .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR, }; @@ -222,7 +258,7 @@ create_image(struct vk_bundle *vk, const struct xrt_swapchain_create_info *info, VkExportMemoryAllocateInfo export_alloc_info = { .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO_KHR, - .pNext = &dedicated_memory_info, + .pNext = use_dedicated_allocation ? &dedicated_memory_info : NULL, .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID, }; @@ -230,8 +266,8 @@ create_image(struct vk_bundle *vk, const struct xrt_swapchain_create_info *info, VkExportMemoryAllocateInfo export_alloc_info = { .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO_KHR, - .pNext = &dedicated_memory_info, - .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT, + .pNext = use_dedicated_allocation ? &dedicated_memory_info : NULL, + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT_KHR, }; #else @@ -248,6 +284,7 @@ create_image(struct vk_bundle *vk, const struct xrt_swapchain_create_info *info, out_image->handle = image; out_image->memory = device_memory; out_image->size = size; + out_image->use_dedicated_allocation = use_dedicated_allocation; return ret; } diff --git a/src/xrt/auxiliary/vk/vk_image_allocator.h b/src/xrt/auxiliary/vk/vk_image_allocator.h index d6c9d1308..864fd9e8f 100644 --- a/src/xrt/auxiliary/vk/vk_image_allocator.h +++ b/src/xrt/auxiliary/vk/vk_image_allocator.h @@ -19,7 +19,7 @@ extern "C" { /*! - * @ingroup aux_vk + * @addtogroup aux_vk * @{ */ @@ -28,6 +28,7 @@ struct vk_image VkImage handle; VkDeviceMemory memory; VkDeviceSize size; + bool use_dedicated_allocation; }; struct vk_image_collection diff --git a/src/xrt/compositor/CMakeLists.txt b/src/xrt/compositor/CMakeLists.txt index 94fcfa548..0913f7efe 100644 --- a/src/xrt/compositor/CMakeLists.txt +++ b/src/xrt/compositor/CMakeLists.txt @@ -1,7 +1,10 @@ -# Copyright 2019-2020, Collabora, Ltd. +# Copyright 2019-2021, Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 spirv_shaders(SHADER_HEADERS + shaders/clear.comp + shaders/distortion.comp + shaders/distortion_timewarp.comp shaders/mesh.frag shaders/mesh.vert shaders/layer.frag @@ -24,6 +27,7 @@ set(MAIN_SOURCE_FILES main/comp_settings.h main/comp_shaders.c main/comp_swapchain.c + main/comp_sync.c main/comp_target.h main/comp_target_swapchain.c main/comp_target_swapchain.h @@ -33,11 +37,20 @@ set(MAIN_SOURCE_FILES main/comp_layer_renderer.h main/comp_layer_renderer.c render/comp_buffer.c + render/comp_compute.c render/comp_render.h render/comp_rendering.c render/comp_resources.c ) +set(MULTI_SOURCE_FILES + multi/comp_multi_compositor.c + multi/comp_multi_interface.h + multi/comp_multi_private.h + multi/comp_multi_system.c + ) + + ### # Client library # @@ -78,7 +91,8 @@ if(XRT_HAVE_OPENGL AND XRT_HAVE_XLIB) endif() if(XRT_HAVE_EGL) list(APPEND CLIENT_SOURCE_FILES - client/comp_egl_glue.c + client/comp_egl_client.c + client/comp_egl_client.h ) endif() @@ -96,6 +110,8 @@ endif() if(XRT_HAVE_OPENGL AND XRT_HAVE_XLIB) target_link_libraries(comp_client PRIVATE OpenGL::GLX) endif() + + ## # Main library # @@ -109,7 +125,6 @@ if(XRT_FEATURE_COMPOSITOR_MAIN) endif() if(XRT_HAVE_XCB AND XRT_HAVE_XLIB) list(APPEND MAIN_SOURCE_FILES - main/comp_window_direct.c main/comp_window_direct_randr.c main/comp_window_direct_nvidia.c ) @@ -124,33 +139,66 @@ if(XRT_FEATURE_COMPOSITOR_MAIN) main/comp_window_vk_display.c ) endif() + if (VK_USE_PLATFORM_DISPLAY_KHR OR XRT_HAVE_XCB) + list(APPEND MAIN_SOURCE_FILES + main/comp_window_direct.c + ) + endif() # generate wayland protocols if(XRT_HAVE_WAYLAND) pkg_get_variable(WL_PROTOS_PKG_DIR wayland-protocols pkgdatadir) pkg_get_variable(WL_SCANNER wayland-scanner wayland_scanner) - set(WL_PROTOS_DIR "${CMAKE_CURRENT_BINARY_DIR}/wayland-protocols/") + set(WL_PROTOS_DIR "${CMAKE_CURRENT_BINARY_DIR}/wayland-protocols") file(MAKE_DIRECTORY "${WL_PROTOS_DIR}") - set(WL_PROTOS_XML "${WL_PROTOS_PKG_DIR}/stable/xdg-shell/xdg-shell.xml") - set(WL_PROTOS_C "${WL_PROTOS_DIR}/xdg-shell.c") - set(WL_PROTOS_H "${WL_PROTOS_DIR}/xdg-shell-client-protocol.h") + set(WL_PROTOS_XML + ${WL_PROTOS_PKG_DIR}/stable/xdg-shell/xdg-shell.xml + ) - add_custom_command( - COMMAND - ${WL_SCANNER} private-code "${WL_PROTOS_XML}" "${WL_PROTOS_C}" - OUTPUT "${WL_PROTOS_C}" VERBATIM) - - add_custom_command( - COMMAND - ${WL_SCANNER} client-header "${WL_PROTOS_XML}" "${WL_PROTOS_H}" - OUTPUT "${WL_PROTOS_H}" VERBATIM) - - set(WL_PROTOS_SRC ${WL_PROTOS_C} ${WL_PROTOS_H}) list(APPEND MAIN_SOURCE_FILES main/comp_window_wayland.c ) + + if (XRT_HAVE_WAYLAND_DIRECT) + list(APPEND WL_PROTOS_XML + ${WL_PROTOS_PKG_DIR}/staging/drm-lease/drm-lease-v1.xml + ) + + list(APPEND MAIN_SOURCE_FILES + main/comp_window_direct_wayland.c + ) + + pkg_check_modules(LIBDRM IMPORTED_TARGET libdrm) + set(WAYLAND_DEPS + ${WAYLAND_LIBRARIES} + PkgConfig::LIBDRM + ) + endif() + + foreach(WL_PROTO_XML ${WL_PROTOS_XML}) + get_filename_component(WL_PROTO ${WL_PROTO_XML} NAME_WE) + + set(WL_PROTO_C "${WL_PROTOS_DIR}/${WL_PROTO}.c") + set(WL_PROTO_H "${WL_PROTOS_DIR}/${WL_PROTO}-client-protocol.h") + + add_custom_command( + COMMAND + ${WL_SCANNER} private-code "${WL_PROTO_XML}" "${WL_PROTO_C}" + OUTPUT "${WL_PROTO_C}" VERBATIM) + + add_custom_command( + COMMAND + ${WL_SCANNER} client-header "${WL_PROTO_XML}" "${WL_PROTO_H}" + OUTPUT "${WL_PROTO_H}" VERBATIM) + + list(APPEND MAIN_SOURCE_FILES + ${WL_PROTO_C} + ${WL_PROTO_H} + ) + endforeach() + endif() if(ANDROID) list(APPEND MAIN_SOURCE_FILES @@ -158,7 +206,7 @@ if(XRT_FEATURE_COMPOSITOR_MAIN) ) endif() - add_library(comp_main STATIC ${SHADER_HEADERS} ${MAIN_SOURCE_FILES} ${WL_PROTOS_SRC}) + add_library(comp_main STATIC ${SHADER_HEADERS} ${MAIN_SOURCE_FILES}) target_link_libraries(comp_main PUBLIC xrt-interfaces PRIVATE aux_util aux_os aux_vk) target_include_directories(comp_main PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(comp_main SYSTEM PRIVATE @@ -168,7 +216,9 @@ if(XRT_FEATURE_COMPOSITOR_MAIN) if(XRT_HAVE_WAYLAND) target_include_directories(comp_main SYSTEM PRIVATE ${WL_PROTOS_DIR}) - target_link_libraries(comp_main PRIVATE ${WAYLAND_LIBRARIES}) + target_link_libraries(comp_main PRIVATE + ${WAYLAND_DEPS} ${WAYLAND_LIBRARIES} + ) endif() if(XRT_HAVE_XCB) target_include_directories(comp_main SYSTEM PRIVATE ${XCB_INCLUDE_DIRS}) @@ -188,3 +238,17 @@ if(XRT_FEATURE_COMPOSITOR_MAIN) add_subdirectory(shaders) endif() + + +### +# Multi client compositor library +# + +add_library(comp_multi STATIC ${MULTI_SOURCE_FILES}) +target_link_libraries(comp_multi PUBLIC xrt-interfaces PRIVATE aux_util aux_os) +target_include_directories(comp_multi PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + + +if(XRT_FEATURE_COMPOSITOR_MAIN) + target_link_libraries(comp_main PRIVATE comp_multi) +endif() diff --git a/src/xrt/compositor/client/comp_egl_glue.c b/src/xrt/compositor/client/comp_egl_client.c similarity index 77% rename from src/xrt/compositor/client/comp_egl_glue.c rename to src/xrt/compositor/client/comp_egl_client.c index 7635d0b7c..f8885397b 100644 --- a/src/xrt/compositor/client/comp_egl_glue.c +++ b/src/xrt/compositor/client/comp_egl_client.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -22,6 +22,7 @@ #include "ogl/ogl_api.h" #include "client/comp_gl_client.h" +#include "client/comp_egl_client.h" #include "client/comp_gl_memobj_swapchain.h" #include "client/comp_gl_eglimage_swapchain.h" @@ -32,6 +33,13 @@ #error "This file shouldn't be compiled without EGL" #endif + +/* + * + * Logging. + * + */ + static enum u_logging_level ll; #define EGL_TRACE(...) U_LOG_IFL_T(ll, __VA_ARGS__) @@ -43,6 +51,12 @@ static enum u_logging_level ll; DEBUG_GET_ONCE_LOG_OPTION(egl_log, "EGL_LOG", U_LOGGING_INFO) +/* + * + * Declarations. + * + */ + #ifdef XRT_OS_ANDROID typedef const char *EGLAPIENTRY (*PFNEGLQUERYSTRINGIMPLEMENTATIONANDROIDPROC)(EGLDisplay dpy, EGLint name); #endif @@ -51,16 +65,57 @@ typedef const char *EGLAPIENTRY (*PFNEGLQUERYSTRINGIMPLEMENTATIONANDROIDPROC)(EG typedef EGLBoolean EGLAPIENTRY (*PFNEGLMAKECURRENTPROC)(EGLDisplay dpy, EGLSurface draw, EGLSurface read, EGLContext ctx); -/*! - * EGL based compositor. - */ -struct client_egl_compositor -{ - struct client_gl_compositor base; +/* + * + * Old helper. + * + */ + +struct old_helper +{ EGLDisplay dpy; + EGLContext ctx; + EGLSurface read, draw; }; +static inline struct old_helper +old_save(void) +{ + struct old_helper old = { + .dpy = eglGetCurrentDisplay(), + .ctx = EGL_NO_CONTEXT, + .read = EGL_NO_SURFACE, + .draw = EGL_NO_SURFACE, + }; + + // Do we have a valid display? + if (old.dpy != EGL_NO_DISPLAY) { + old.ctx = eglGetCurrentContext(); + old.read = eglGetCurrentSurface(EGL_READ); + old.draw = eglGetCurrentSurface(EGL_DRAW); + } + + return old; +} + +static inline void +old_restore(struct old_helper *old, EGLDisplay current_dpy) +{ + if (old->dpy == EGL_NO_DISPLAY) { + // There were no display, just unbind the context. + if (eglMakeCurrent(current_dpy, EGL_NO_CONTEXT, EGL_NO_SURFACE, EGL_NO_SURFACE)) { + return; + } + } else { + if (eglMakeCurrent(old->dpy, old->draw, old->read, old->ctx)) { + return; + } + } + + EGL_ERROR("Failed to make old EGL context current! (%p, %p, %p, %p)", old->dpy, old->draw, old->read, old->ctx); +} + /* * @@ -68,16 +123,6 @@ struct client_egl_compositor * */ -/*! - * Down-cast helper. - * @protected @memberof client_egl_compositor - */ -static inline struct client_egl_compositor * -client_egl_compositor(struct xrt_compositor *xc) -{ - return (struct client_egl_compositor *)xc; -} - XRT_MAYBE_UNUSED static bool has_extension(const char *extensions, const char *ext) { @@ -127,43 +172,6 @@ ensure_native_fence_is_loaded(EGLDisplay dpy, PFNEGLGETPROCADDRESSPROC get_gl_pr } -/* - * - * Old helper. - * - */ - -struct old_helper -{ - EGLDisplay dpy; - EGLContext ctx; - EGLSurface read, draw; -}; - -static inline struct old_helper -old_save(void) -{ - struct old_helper old = { - .dpy = eglGetCurrentDisplay(), - .ctx = eglGetCurrentContext(), - .read = eglGetCurrentSurface(EGL_READ), - .draw = eglGetCurrentSurface(EGL_DRAW), - }; - - return old; -} - -static inline void -old_restore(struct old_helper *old) -{ - if (eglMakeCurrent(old->dpy, old->draw, old->read, old->ctx)) { - return; - } - - EGL_ERROR("Failed to make old EGL context current! (%p, %p, %p, %p)", old->dpy, old->draw, old->read, old->ctx); -} - - /* * * Functions. @@ -201,6 +209,7 @@ insert_fence(struct xrt_compositor *xc, xrt_graphics_sync_handle_t *out_handle) } *out_handle = fence_fd; + #else (void)cglc; #endif @@ -216,17 +225,23 @@ client_egl_compositor_destroy(struct xrt_compositor *xc) free(ceglc); } -struct xrt_compositor_gl * +xrt_result_t xrt_gfx_provider_create_gl_egl(struct xrt_compositor_native *xcn, EGLDisplay display, EGLConfig config, EGLContext context, - PFNEGLGETPROCADDRESSPROC get_gl_procaddr) + PFNEGLGETPROCADDRESSPROC get_gl_procaddr, + struct xrt_compositor_gl **out_xcgl) { ll = debug_get_log_option_egl_log(); gladLoadEGL(display, get_gl_procaddr); + if (config == EGL_NO_CONFIG_KHR && !EGL_KHR_no_config_context) { + EGL_ERROR("config == EGL_NO_CONFIG_KHR && !EGL_KHR_no_config_context"); + return XRT_ERROR_EGL_CONFIG_MISSING; + } + // On Android this extension is 'hidden'. ensure_native_fence_is_loaded(display, get_gl_procaddr); @@ -236,13 +251,13 @@ xrt_gfx_provider_create_gl_egl(struct xrt_compositor_native *xcn, if (!eglMakeCurrent(display, EGL_NO_SURFACE, EGL_NO_SURFACE, context)) { EGL_ERROR("Failed to make EGL context current"); // No need to restore on failure. - return NULL; + return XRT_ERROR_OPENGL; } EGLint egl_client_type; if (!eglQueryContext(display, context, EGL_CONTEXT_CLIENT_TYPE, &egl_client_type)) { - old_restore(&old); - return NULL; + old_restore(&old, display); + return XRT_ERROR_OPENGL; } switch (egl_client_type) { @@ -252,8 +267,8 @@ xrt_gfx_provider_create_gl_egl(struct xrt_compositor_native *xcn, break; #else EGL_ERROR("OpenGL support not including in this runtime build"); - old_restore(&old); - return NULL; + old_restore(&old, display); + return XRT_ERROR_OPENGL; #endif case EGL_OPENGL_ES_API: @@ -262,10 +277,10 @@ xrt_gfx_provider_create_gl_egl(struct xrt_compositor_native *xcn, break; #else EGL_ERROR("OpenGL|ES support not including in this runtime build"); - old_restore(&old); - return NULL; + old_restore(&old, display); + return XRT_ERROR_OPENGL; #endif - default: EGL_ERROR("Unsupported EGL client type"); return NULL; + default: EGL_ERROR("Unsupported EGL client type"); return XRT_ERROR_OPENGL; } struct client_egl_compositor *ceglc = U_TYPED_CALLOC(struct client_egl_compositor); @@ -273,8 +288,8 @@ xrt_gfx_provider_create_gl_egl(struct xrt_compositor_native *xcn, client_gl_swapchain_create_func sc_create = NULL; - EGL_INFO("Extension availability:"); -#define DUMP_EXTENSION_STATUS(EXT) EGL_INFO(" - " #EXT ": %s", GLAD_##EXT ? "true" : "false") + EGL_DEBUG("Extension availability:"); +#define DUMP_EXTENSION_STATUS(EXT) EGL_DEBUG(" - " #EXT ": %s", GLAD_##EXT ? "true" : "false") DUMP_EXTENSION_STATUS(GL_EXT_memory_object); DUMP_EXTENSION_STATUS(GL_EXT_memory_object_fd); @@ -290,38 +305,46 @@ xrt_gfx_provider_create_gl_egl(struct xrt_compositor_native *xcn, DUMP_EXTENSION_STATUS(EGL_KHR_reusable_sync); DUMP_EXTENSION_STATUS(EGL_KHR_wait_sync); +#undef DUMP_EXTENSION_STATUS #if defined(XRT_GRAPHICS_BUFFER_HANDLE_IS_FD) + if (GLAD_GL_EXT_memory_object && GLAD_GL_EXT_memory_object_fd) { - EGL_INFO("Using GL memory object swapchain implementation"); + EGL_DEBUG("Using GL memory object swapchain implementation"); sc_create = client_gl_memobj_swapchain_create; } + if (sc_create == NULL && GLAD_EGL_EXT_image_dma_buf_import) { - EGL_INFO("Using EGL_Image swapchain implementation"); + EGL_DEBUG("Using EGL_Image swapchain implementation"); sc_create = client_gl_eglimage_swapchain_create; } + if (sc_create == NULL) { free(ceglc); EGL_ERROR( - "Could not find a required extension: need either " - "EGL_EXT_image_dma_buf_import or " + "Could not find a required extension: need either EGL_EXT_image_dma_buf_import or " "GL_EXT_memory_object_fd"); - old_restore(&old); - return NULL; + old_restore(&old, display); + return XRT_ERROR_OPENGL; } + #elif defined(XRT_GRAPHICS_BUFFER_HANDLE_IS_AHARDWAREBUFFER) - EGL_INFO("Using EGL_Image swapchain implementation with AHardwareBuffer"); + + EGL_DEBUG("Using EGL_Image swapchain implementation with AHardwareBuffer"); sc_create = client_gl_eglimage_swapchain_create; + #endif if (!client_gl_compositor_init(&ceglc->base, xcn, sc_create, insert_fence)) { free(ceglc); - U_LOG_E("Failed to initialize compositor"); - old_restore(&old); - return NULL; + EGL_ERROR("Failed to initialize compositor"); + old_restore(&old, display); + return XRT_ERROR_OPENGL; } ceglc->base.base.base.destroy = client_egl_compositor_destroy; - old_restore(&old); - return &ceglc->base.base; + old_restore(&old, display); + *out_xcgl = &ceglc->base.base; + + return XRT_SUCCESS; } diff --git a/src/xrt/compositor/client/comp_egl_client.h b/src/xrt/compositor/client/comp_egl_client.h new file mode 100644 index 000000000..14804ed7b --- /dev/null +++ b/src/xrt/compositor/client/comp_egl_client.h @@ -0,0 +1,50 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Glue code to EGL client side glue code. + * @author Jakob Bornecrantz + * @ingroup comp_client + */ + +#pragma once + +#include "xrt/xrt_compositor.h" + +#include "ogl/egl_api.h" + +#include "client/comp_gl_client.h" + + +#ifdef __cplusplus +extern "C" { +#endif + + +/*! + * EGL based compositor, carries the extra needed EGL information needed by the + * client side code and can handle both GL Desktop or GLES contexts. + * + * @ingroup comp_client + */ +struct client_egl_compositor +{ + struct client_gl_compositor base; + + EGLDisplay dpy; +}; + +/*! + * Down-cast helper. + * @protected @memberof client_egl_compositor + */ +static inline struct client_egl_compositor * +client_egl_compositor(struct xrt_compositor *xc) +{ + return (struct client_egl_compositor *)xc; +} + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/compositor/client/comp_gl_client.c b/src/xrt/compositor/client/comp_gl_client.c index 0fd2a0d4e..f5445bb00 100644 --- a/src/xrt/compositor/client/comp_gl_client.c +++ b/src/xrt/compositor/client/comp_gl_client.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -130,11 +130,14 @@ client_gl_compositor_discard_frame(struct xrt_compositor *xc, int64_t frame_id) } static xrt_result_t -client_gl_compositor_layer_begin(struct xrt_compositor *xc, int64_t frame_id, enum xrt_blend_mode env_blend_mode) +client_gl_compositor_layer_begin(struct xrt_compositor *xc, + int64_t frame_id, + uint64_t display_time_ns, + enum xrt_blend_mode env_blend_mode) { struct client_gl_compositor *c = client_gl_compositor(xc); - return xrt_comp_layer_begin(&c->xcn->base, frame_id, env_blend_mode); + return xrt_comp_layer_begin(&c->xcn->base, frame_id, display_time_ns, env_blend_mode); } static xrt_result_t @@ -293,8 +296,7 @@ client_gl_compositor_layer_commit(struct xrt_compositor *xc, int64_t frame_id, x xret = c->insert_fence(xc, &sync_handle); } else { /*! - * @hack: The swapchain images should have been externally - * synchronized. + * @todo The swapchain images should have been externally synchronized. */ glFlush(); } @@ -310,9 +312,14 @@ static int64_t gl_format_to_vk(int64_t format) { switch (format) { - case GL_RGBA8: return 37 /*VK_FORMAT_R8G8B8A8_UNORM*/; + case GL_RGB8: return 23 /*VK_FORMAT_R8G8B8_UNORM*/; // Should not be used, colour precision. + case GL_SRGB8: return 29 /*VK_FORMAT_R8G8B8_SRGB*/; + case GL_RGBA8: return 37 /*VK_FORMAT_R8G8B8A8_UNORM*/; // Should not be used, colour precision. case GL_SRGB8_ALPHA8: return 43 /*VK_FORMAT_R8G8B8A8_SRGB*/; case GL_RGB10_A2: return 64 /*VK_FORMAT_A2B10G10R10_UNORM_PACK32*/; + case GL_RGB16: return 84 /*VK_FORMAT_R16G16B16_UNORM*/; + case GL_RGB16F: return 90 /*VK_FORMAT_R16G16B16_SFLOAT*/; + case GL_RGBA16: return 91 /*VK_FORMAT_R16G16B16A16_UNORM*/; case GL_RGBA16F: return 97 /*VK_FORMAT_R16G16B16A16_SFLOAT*/; case GL_DEPTH_COMPONENT16: return 124 /*VK_FORMAT_D16_UNORM*/; case GL_DEPTH_COMPONENT32F: return 126 /*VK_FORMAT_D32_SFLOAT*/; @@ -326,17 +333,23 @@ static int64_t vk_format_to_gl(int64_t format) { switch (format) { - case 37 /*VK_FORMAT_R8G8B8A8_UNORM*/: return GL_RGBA8; + case 23 /*VK_FORMAT_R8G8B8_UNORM*/: return GL_RGB8; // Should not be used, colour precision. + case 29 /*VK_FORMAT_R8G8B8_SRGB*/: return GL_SRGB8; + case 30 /*VK_FORMAT_B8G8R8_UNORM*/: return 0; + case 37 /*VK_FORMAT_R8G8B8A8_UNORM*/: return GL_RGBA8; // Should not be used, colour precision. case 43 /*VK_FORMAT_R8G8B8A8_SRGB*/: return GL_SRGB8_ALPHA8; case 44 /*VK_FORMAT_B8G8R8A8_UNORM*/: return 0; case 50 /*VK_FORMAT_B8G8R8A8_SRGB*/: return 0; case 64 /*VK_FORMAT_A2B10G10R10_UNORM_PACK32*/: return GL_RGB10_A2; + case 84 /*VK_FORMAT_R16G16B16_UNORM*/: return GL_RGB16; + case 90 /*VK_FORMAT_R16G16B16_SFLOAT*/: return GL_RGB16F; + case 91 /*VK_FORMAT_R16G16B16A16_UNORM*/: return GL_RGBA16; case 97 /*VK_FORMAT_R16G16B16A16_SFLOAT*/: return GL_RGBA16F; case 124 /*VK_FORMAT_D16_UNORM*/: return GL_DEPTH_COMPONENT16; case 126 /*VK_FORMAT_D32_SFLOAT*/: return GL_DEPTH_COMPONENT32F; case 129 /*VK_FORMAT_D24_UNORM_S8_UINT*/: return GL_DEPTH24_STENCIL8; case 130 /*VK_FORMAT_D32_SFLOAT_S8_UINT*/: return GL_DEPTH32F_STENCIL8; - default: U_LOG_W("Cannot convert VK format 0x%016" PRIx64 " to GL format!\n", format); return 0; + default: U_LOG_W("Cannot convert VK format %" PRIu64 " to GL format!\n", format); return 0; } } @@ -366,7 +379,7 @@ client_gl_swapchain_create(struct xrt_compositor *xc, struct xrt_swapchain_create_info xinfo = *info; xinfo.format = vk_format; - struct xrt_swapchain_native *xscn = NULL; + struct xrt_swapchain_native *xscn = NULL; // Has to be NULL. xret = xrt_comp_native_create_swapchain(c->xcn, &xinfo, &xscn); @@ -387,7 +400,8 @@ client_gl_swapchain_create(struct xrt_compositor *xc, struct client_gl_swapchain *sc = NULL; if (NULL == c->create_swapchain(xc, info, xscn, &sc)) { - xrt_swapchain_destroy(&xsc); + // Drop our reference, does NULL checking. + xrt_swapchain_reference(&xsc, NULL); return XRT_ERROR_OPENGL; } diff --git a/src/xrt/compositor/client/comp_gl_client.h b/src/xrt/compositor/client/comp_gl_client.h index 84742c6e0..caf2dfb41 100644 --- a/src/xrt/compositor/client/comp_gl_client.h +++ b/src/xrt/compositor/client/comp_gl_client.h @@ -125,7 +125,7 @@ client_gl_compositor(struct xrt_compositor *xc) * won't be called for you. * * @public @memberof client_gl_compositor - * @relatesalso xrt_compositor_native + * @see xrt_compositor_native */ bool client_gl_compositor_init(struct client_gl_compositor *c, diff --git a/src/xrt/compositor/client/comp_gl_eglimage_swapchain.c b/src/xrt/compositor/client/comp_gl_eglimage_swapchain.c index 977373c3c..797e6cac7 100644 --- a/src/xrt/compositor/client/comp_gl_eglimage_swapchain.c +++ b/src/xrt/compositor/client/comp_gl_eglimage_swapchain.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -27,6 +27,7 @@ #include "ogl/ogl_helpers.h" #include "client/comp_gl_client.h" +#include "client/comp_egl_client.h" #include "client/comp_gl_eglimage_swapchain.h" #include @@ -84,8 +85,8 @@ client_gl_eglimage_swapchain_destroy(struct xrt_swapchain *xsc) client_gl_eglimage_swapchain_teardown_storage(sc); sc->base.base.base.num_images = 0; - // Destroy the native swapchain as well. - xrt_swapchain_destroy((struct xrt_swapchain **)&sc->base.xscn); + // Drop our reference, does NULL checking. + xrt_swapchain_native_reference(&sc->base.xscn, NULL); free(sc); } @@ -164,6 +165,7 @@ client_gl_eglimage_swapchain_create(struct xrt_compositor *xc, struct xrt_swapchain_native *xscn, struct client_gl_swapchain **out_sc) { + struct client_egl_compositor *ceglc = client_egl_compositor(xc); ll = debug_get_log_option_egl_swapchain_log(); if (xscn == NULL) { @@ -193,14 +195,13 @@ client_gl_eglimage_swapchain_create(struct xrt_compositor *xc, struct xrt_swapchain *native_xsc = &xscn->base; struct client_gl_eglimage_swapchain *sc = U_TYPED_CALLOC(struct client_gl_eglimage_swapchain); - struct xrt_swapchain_gl *xscgl = &sc->base.base; - struct xrt_swapchain *client_xsc = &xscgl->base; - client_xsc->destroy = client_gl_eglimage_swapchain_destroy; - // Fetch the number of images from the native swapchain. - client_xsc->num_images = native_xsc->num_images; + sc->base.base.base.destroy = client_gl_eglimage_swapchain_destroy; + sc->base.base.base.reference.count = 1; + sc->base.base.base.num_images = native_xsc->num_images; // Fetch the number of images from the native swapchain. sc->base.xscn = xscn; + sc->display = ceglc->dpy; - sc->display = eglGetCurrentDisplay(); + struct xrt_swapchain_gl *xscgl = &sc->base.base; glGenTextures(native_xsc->num_images, xscgl->images); diff --git a/src/xrt/compositor/client/comp_gl_memobj_swapchain.c b/src/xrt/compositor/client/comp_gl_memobj_swapchain.c index 8e4a060b6..53be1b8f6 100644 --- a/src/xrt/compositor/client/comp_gl_memobj_swapchain.c +++ b/src/xrt/compositor/client/comp_gl_memobj_swapchain.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -55,11 +55,12 @@ client_gl_memobj_swapchain_destroy(struct xrt_swapchain *xsc) sc->base.base.base.num_images = 0; } - // Destroy the native swapchain as well. - xrt_swapchain_destroy((struct xrt_swapchain **)&sc->base.xscn); + // Drop our reference, does NULL checking. + xrt_swapchain_reference((struct xrt_swapchain **)&sc->base.xscn, NULL); free(sc); } + struct xrt_swapchain * client_gl_memobj_swapchain_create(struct xrt_compositor *xc, const struct xrt_swapchain_create_info *info, @@ -80,22 +81,21 @@ client_gl_memobj_swapchain_create(struct xrt_compositor *xc, struct xrt_swapchain *native_xsc = &xscn->base; struct client_gl_memobj_swapchain *sc = U_TYPED_CALLOC(struct client_gl_memobj_swapchain); - struct xrt_swapchain_gl *xscgl = &sc->base.base; - struct xrt_swapchain *client_xsc = &xscgl->base; - client_xsc->destroy = client_gl_memobj_swapchain_destroy; - // Fetch the number of images from the native swapchain. - client_xsc->num_images = native_xsc->num_images; + sc->base.base.base.destroy = client_gl_memobj_swapchain_destroy; + sc->base.base.base.reference.count = 1; + sc->base.base.base.num_images = native_xsc->num_images; // Fetch the number of images from the native swapchain. sc->base.xscn = xscn; sc->base.tex_target = tex_target; + struct xrt_swapchain_gl *xscgl = &sc->base.base; + glGenTextures(native_xsc->num_images, xscgl->images); - for (uint32_t i = 0; i < native_xsc->num_images; i++) { - glBindTexture(tex_target, xscgl->images[i]); - } glCreateMemoryObjectsEXT(native_xsc->num_images, &sc->memory[0]); for (uint32_t i = 0; i < native_xsc->num_images; i++) { - GLint dedicated = GL_TRUE; + glBindTexture(tex_target, xscgl->images[i]); + + GLint dedicated = xscn->images[i].use_dedicated_allocation ? GL_TRUE : GL_FALSE; glMemoryObjectParameterivEXT(sc->memory[i], GL_DEDICATED_MEMORY_OBJECT_EXT, &dedicated); glImportMemoryFdEXT(sc->memory[i], xscn->images[i].size, GL_HANDLE_TYPE_OPAQUE_FD_EXT, xscn->images[i].handle); @@ -113,7 +113,7 @@ client_gl_memobj_swapchain_create(struct xrt_compositor *xc, } *out_cglsc = &sc->base; - return client_xsc; + return &sc->base.base.base; #else // silence unused function warning diff --git a/src/xrt/compositor/client/comp_gl_xlib_client.h b/src/xrt/compositor/client/comp_gl_xlib_client.h index a7435fcd7..aa10c5013 100644 --- a/src/xrt/compositor/client/comp_gl_xlib_client.h +++ b/src/xrt/compositor/client/comp_gl_xlib_client.h @@ -34,7 +34,7 @@ struct client_gl_xlib_compositor * Create a new client_gl_xlib_compositor. * * @public @memberof client_gl_xlib_compositor - * @relatesalso xrt_compositor_native + * @see xrt_compositor_native */ struct client_gl_xlib_compositor * client_gl_xlib_compositor_create(struct xrt_compositor_native *xcn, diff --git a/src/xrt/compositor/client/comp_vk_client.c b/src/xrt/compositor/client/comp_vk_client.c index 3e2e8c62a..268b48750 100644 --- a/src/xrt/compositor/client/comp_vk_client.c +++ b/src/xrt/compositor/client/comp_vk_client.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -16,6 +16,8 @@ #include #include +#define MS_TO_NS(ms) (ms * 1000L * 1000L) + /*! * Down-cast helper. * @@ -49,21 +51,29 @@ client_vk_swapchain_destroy(struct xrt_swapchain *xsc) { struct client_vk_swapchain *sc = client_vk_swapchain(xsc); struct client_vk_compositor *c = sc->c; + struct vk_bundle *vk = &c->vk; for (uint32_t i = 0; i < sc->base.base.num_images; i++) { + + VkResult ret = vk->vkWaitForFences(vk->device, 1, &sc->acquire_release_fence[i], true, MS_TO_NS(500)); + if (vk_has_error(ret, "vkWaitForFences", __FILE__, __LINE__)) { + // don't really care, we are going to destroy anyway, just make sure it's not used anymore + vk->vkDeviceWaitIdle(vk->device); + } + if (sc->base.images[i] != VK_NULL_HANDLE) { - c->vk.vkDestroyImage(c->vk.device, sc->base.images[i], NULL); + vk->vkDestroyImage(vk->device, sc->base.images[i], NULL); sc->base.images[i] = VK_NULL_HANDLE; } if (sc->mems[i] != VK_NULL_HANDLE) { - c->vk.vkFreeMemory(c->vk.device, sc->mems[i], NULL); + vk->vkFreeMemory(vk->device, sc->mems[i], NULL); sc->mems[i] = VK_NULL_HANDLE; } } - // Destroy the native swapchain as well. - xrt_swapchain_destroy((struct xrt_swapchain **)&sc->xscn); + // Drop our reference, does NULL checking. + xrt_swapchain_native_reference(&sc->xscn, NULL); free(sc); } @@ -80,6 +90,14 @@ client_vk_swapchain_acquire_image(struct xrt_swapchain *xsc, uint32_t *out_index return xret; } + VkResult ret; + + ret = vk->vkWaitForFences(vk->device, 1, &sc->acquire_release_fence[*out_index], true, MS_TO_NS(500)); + vk_check_error("vkWaitForFences", ret, XRT_ERROR_VULKAN); + + ret = vk->vkResetFences(vk->device, 1, &sc->acquire_release_fence[*out_index]); + vk_check_error("vkResetFences", ret, XRT_ERROR_VULKAN); + // Acquire ownership and complete layout transition VkSubmitInfo submitInfo = { .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, @@ -87,11 +105,12 @@ client_vk_swapchain_acquire_image(struct xrt_swapchain *xsc, uint32_t *out_index .pCommandBuffers = &sc->acquire[*out_index], }; - VkResult ret = vk_locked_submit(vk, vk->queue, 1, &submitInfo, VK_NULL_HANDLE); + ret = vk_locked_submit(vk, vk->queue, 1, &submitInfo, sc->acquire_release_fence[*out_index]); if (ret != VK_SUCCESS) { VK_ERROR(vk, "Could not submit to queue: %d", ret); return XRT_ERROR_FAILED_TO_SUBMIT_VULKAN_COMMANDS; } + return XRT_SUCCESS; } @@ -110,6 +129,14 @@ client_vk_swapchain_release_image(struct xrt_swapchain *xsc, uint32_t index) struct client_vk_swapchain *sc = client_vk_swapchain(xsc); struct vk_bundle *vk = &sc->c->vk; + VkResult ret; + + ret = vk->vkWaitForFences(vk->device, 1, &sc->acquire_release_fence[index], true, MS_TO_NS(500)); + vk_check_error("vkWaitForFences", ret, XRT_ERROR_VULKAN); + + vk->vkResetFences(vk->device, 1, &sc->acquire_release_fence[index]); + vk_check_error("vkResetFences", ret, XRT_ERROR_VULKAN); + // Release ownership and begin layout transition VkSubmitInfo submitInfo = { .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, @@ -117,7 +144,7 @@ client_vk_swapchain_release_image(struct xrt_swapchain *xsc, uint32_t index) .pCommandBuffers = &sc->release[index], }; - VkResult ret = vk_locked_submit(vk, vk->queue, 1, &submitInfo, VK_NULL_HANDLE); + ret = vk_locked_submit(vk, vk->queue, 1, &submitInfo, sc->acquire_release_fence[index]); if (ret != VK_SUCCESS) { VK_ERROR(vk, "Could not submit to queue: %d", ret); return XRT_ERROR_FAILED_TO_SUBMIT_VULKAN_COMMANDS; @@ -147,17 +174,19 @@ static void client_vk_compositor_destroy(struct xrt_compositor *xc) { struct client_vk_compositor *c = client_vk_compositor(xc); + struct vk_bundle *vk = &c->vk; - if (c->vk.cmd_pool != VK_NULL_HANDLE) { + if (vk->cmd_pool != VK_NULL_HANDLE) { // Make sure that any of the command buffers from this command // pool are n used here, this pleases the validation layer. - os_mutex_lock(&c->vk.queue_mutex); - c->vk.vkDeviceWaitIdle(c->vk.device); - os_mutex_unlock(&c->vk.queue_mutex); + os_mutex_lock(&vk->queue_mutex); + vk->vkDeviceWaitIdle(vk->device); + os_mutex_unlock(&vk->queue_mutex); - c->vk.vkDestroyCommandPool(c->vk.device, c->vk.cmd_pool, NULL); - c->vk.cmd_pool = VK_NULL_HANDLE; + vk->vkDestroyCommandPool(vk->device, vk->cmd_pool, NULL); + vk->cmd_pool = VK_NULL_HANDLE; } + vk_deinit_mutex(vk); free(c); } @@ -211,11 +240,14 @@ client_vk_compositor_discard_frame(struct xrt_compositor *xc, int64_t frame_id) } static xrt_result_t -client_vk_compositor_layer_begin(struct xrt_compositor *xc, int64_t frame_id, enum xrt_blend_mode env_blend_mode) +client_vk_compositor_layer_begin(struct xrt_compositor *xc, + int64_t frame_id, + uint64_t display_time_ns, + enum xrt_blend_mode env_blend_mode) { struct client_vk_compositor *c = client_vk_compositor(xc); - return xrt_comp_layer_begin(&c->xcn->base, frame_id, env_blend_mode); + return xrt_comp_layer_begin(&c->xcn->base, frame_id, display_time_ns, env_blend_mode); } static xrt_result_t @@ -356,11 +388,12 @@ client_vk_swapchain_create(struct xrt_compositor *xc, struct xrt_swapchain **out_xsc) { struct client_vk_compositor *c = client_vk_compositor(xc); + struct vk_bundle *vk = &c->vk; VkCommandBuffer cmd_buffer; VkResult ret; xrt_result_t xret; - struct xrt_swapchain_native *xscn = NULL; + struct xrt_swapchain_native *xscn = NULL; // Has to be NULL. xret = xrt_comp_native_create_swapchain(c->xcn, info, &xscn); if (xret != XRT_SUCCESS) { @@ -370,7 +403,7 @@ client_vk_swapchain_create(struct xrt_compositor *xc, struct xrt_swapchain *xsc = &xscn->base; - ret = vk_init_cmd_buffer(&c->vk, &cmd_buffer); + ret = vk_init_cmd_buffer(vk, &cmd_buffer); if (ret != VK_SUCCESS) { return XRT_ERROR_VULKAN; } @@ -388,13 +421,13 @@ client_vk_swapchain_create(struct xrt_compositor *xc, sc->base.base.acquire_image = client_vk_swapchain_acquire_image; sc->base.base.wait_image = client_vk_swapchain_wait_image; sc->base.base.release_image = client_vk_swapchain_release_image; - // Fetch the number of images from the native swapchain. - sc->base.base.num_images = xsc->num_images; + sc->base.base.reference.count = 1; + sc->base.base.num_images = xsc->num_images; // Fetch the number of images from the native swapchain. sc->c = c; sc->xscn = xscn; for (uint32_t i = 0; i < xsc->num_images; i++) { - ret = vk_create_image_from_native(&c->vk, info, &xscn->images[i], &sc->base.images[i], &sc->mems[i]); + ret = vk_create_image_from_native(vk, info, &xscn->images[i], &sc->base.images[i], &sc->mems[i]); if (ret != VK_SUCCESS) { @@ -406,11 +439,11 @@ client_vk_swapchain_create(struct xrt_compositor *xc, * not be a bug in the validation layer. That may or may not be * fixed in the future version of the validation layer. */ - vk_set_image_layout(&c->vk, cmd_buffer, sc->base.images[i], 0, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + vk_set_image_layout(vk, cmd_buffer, sc->base.images[i], 0, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, subresource_range); } - ret = vk_submit_cmd_buffer(&c->vk, cmd_buffer); + ret = vk_submit_cmd_buffer(vk, cmd_buffer); if (ret != VK_SUCCESS) { return XRT_ERROR_FAILED_TO_SUBMIT_VULKAN_COMMANDS; } @@ -418,11 +451,11 @@ client_vk_swapchain_create(struct xrt_compositor *xc, // Prerecord command buffers for swapchain image ownership/layout // transitions for (uint32_t i = 0; i < xsc->num_images; i++) { - ret = vk_init_cmd_buffer(&c->vk, &sc->acquire[i]); + ret = vk_init_cmd_buffer(vk, &sc->acquire[i]); if (ret != VK_SUCCESS) { return XRT_ERROR_VULKAN; } - ret = vk_init_cmd_buffer(&c->vk, &sc->release[i]); + ret = vk_init_cmd_buffer(vk, &sc->release[i]); if (ret != VK_SUCCESS) { return XRT_ERROR_VULKAN; } @@ -469,28 +502,34 @@ client_vk_swapchain_create(struct xrt_compositor *xc, .dstAccessMask = 0, .oldLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - .srcQueueFamilyIndex = c->vk.queue_family_index, + .srcQueueFamilyIndex = vk->queue_family_index, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL, .image = sc->base.images[i], .subresourceRange = subresource_range, }; //! @todo less conservative pipeline stage masks based on usage - c->vk.vkCmdPipelineBarrier(sc->acquire[i], VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, - VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, NULL, 0, NULL, 1, &acquire); - c->vk.vkCmdPipelineBarrier(sc->release[i], VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, NULL, 0, NULL, 1, &release); + vk->vkCmdPipelineBarrier(sc->acquire[i], VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, NULL, 0, NULL, 1, &acquire); + vk->vkCmdPipelineBarrier(sc->release[i], VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, NULL, 0, NULL, 1, &release); - ret = c->vk.vkEndCommandBuffer(sc->acquire[i]); + ret = vk->vkEndCommandBuffer(sc->acquire[i]); if (ret != VK_SUCCESS) { - VK_ERROR((&c->vk), "vkEndCommandBuffer: %s", vk_result_string(ret)); + VK_ERROR(vk, "vkEndCommandBuffer: %s", vk_result_string(ret)); return XRT_ERROR_VULKAN; } - ret = c->vk.vkEndCommandBuffer(sc->release[i]); + ret = vk->vkEndCommandBuffer(sc->release[i]); if (ret != VK_SUCCESS) { - VK_ERROR((&c->vk), "vkEndCommandBuffer: %s", vk_result_string(ret)); + VK_ERROR(vk, "vkEndCommandBuffer: %s", vk_result_string(ret)); return XRT_ERROR_VULKAN; } + + VkFenceCreateInfo fence_create_info = { + .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, + .flags = VK_FENCE_CREATE_SIGNALED_BIT, + }; + vk->vkCreateFence(vk->device, &fence_create_info, NULL, &sc->acquire_release_fence[i]); } *out_xsc = &sc->base.base; @@ -541,6 +580,10 @@ client_vk_compositor_create(struct xrt_compositor_native *xcn, goto err_free; } + ret = vk_init_mutex(&c->vk); + if (ret != VK_SUCCESS) { + goto err_free; + } return c; err_free: diff --git a/src/xrt/compositor/client/comp_vk_client.h b/src/xrt/compositor/client/comp_vk_client.h index 375c967e2..33e8e6350 100644 --- a/src/xrt/compositor/client/comp_vk_client.h +++ b/src/xrt/compositor/client/comp_vk_client.h @@ -50,6 +50,7 @@ struct client_vk_swapchain // Prerecorded swapchain image ownership/layout transition barriers VkCommandBuffer acquire[XRT_MAX_SWAPCHAIN_IMAGES]; VkCommandBuffer release[XRT_MAX_SWAPCHAIN_IMAGES]; + VkFence acquire_release_fence[XRT_MAX_SWAPCHAIN_IMAGES]; }; /*! @@ -84,7 +85,7 @@ struct client_vk_compositor * Takes owenership of provided xcn. * * @public @memberof client_vk_compositor - * @relatesalso xrt_compositor_native + * @see xrt_compositor_native */ struct client_vk_compositor * client_vk_compositor_create(struct xrt_compositor_native *xcn, diff --git a/src/xrt/compositor/main/comp_compositor.c b/src/xrt/compositor/main/comp_compositor.c index 0aa8eb3b1..876488c55 100644 --- a/src/xrt/compositor/main/comp_compositor.c +++ b/src/xrt/compositor/main/comp_compositor.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -42,6 +42,7 @@ */ #include "xrt/xrt_gfx_native.h" +#include "xrt/xrt_config_have.h" #include "os/os_time.h" @@ -49,10 +50,14 @@ #include "util/u_misc.h" #include "util/u_time.h" #include "util/u_debug.h" +#include "util/u_handles.h" +#include "util/u_trace_marker.h" #include "util/u_distortion_mesh.h" #include "main/comp_compositor.h" +#include "multi/comp_multi_interface.h" + #include #include #include @@ -74,6 +79,8 @@ * */ +#define CVK_ERROR(C, FUNC, MSG, RET) COMP_ERROR(C, FUNC ": %s\t\n" MSG, vk_result_string(RET)); + static double ns_to_ms(int64_t ns) { @@ -111,116 +118,110 @@ compositor_end_session(struct xrt_compositor *xc) return XRT_SUCCESS; } -/*! - * @brief Utility for waiting (for rendering purposes) until the next vsync or a - * specified time point, whichever comes first. - * - * Only for rendering - this will busy-wait if needed. - * - * @return true if we waited until the time indicated - * - * @todo In the future, this may differ between platforms since some have ways - * to directly wait on a vsync. - */ -static bool -compositor_wait_vsync_or_time(struct comp_compositor *c, int64_t wake_up_time) +static xrt_result_t +compositor_predict_frame(struct xrt_compositor *xc, + int64_t *out_frame_id, + uint64_t *out_wake_time_ns, + uint64_t *out_predicted_gpu_time_ns, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns) { + COMP_TRACE_MARKER(); - int64_t now_ns = os_monotonic_get_ns(); - /*! - * @todo this is not accurate, but it serves the purpose of not letting - * us sleep longer than the next vsync usually - */ - int64_t next_vsync = now_ns + c->settings.nominal_frame_interval_ns / 2; + struct comp_compositor *c = comp_compositor(xc); - bool ret = true; - // Sleep until the sooner of vsync or our deadline. - if (next_vsync < wake_up_time) { - ret = false; - wake_up_time = next_vsync; - } - int64_t wait_duration = wake_up_time - now_ns; - if (wait_duration <= 0) { - // Don't wait at all - return ret; - } + COMP_SPEW(c, "PREDICT_FRAME"); - if (wait_duration > 1000000) { - os_nanosleep(wait_duration - (wait_duration % 1000000)); - } - // Busy-wait for fine-grained delays. - while (now_ns < wake_up_time) { - now_ns = os_monotonic_get_ns(); - } + // A little bit easier to read. + uint64_t interval_ns = (int64_t)c->settings.nominal_frame_interval_ns; - return ret; + comp_target_update_timings(c->target); + + assert(c->frame.waited.id == -1); + + int64_t frame_id = -1; + uint64_t wake_up_time_ns = 0; + uint64_t present_slop_ns = 0; + uint64_t desired_present_time_ns = 0; + uint64_t predicted_display_time_ns = 0; + comp_target_calc_frame_timings( // + c->target, // + &frame_id, // + &wake_up_time_ns, // + &desired_present_time_ns, // + &present_slop_ns, // + &predicted_display_time_ns); // + + c->frame.waited.id = frame_id; + c->frame.waited.desired_present_time_ns = desired_present_time_ns; + c->frame.waited.present_slop_ns = present_slop_ns; + c->frame.waited.predicted_display_time_ns = predicted_display_time_ns; + + *out_frame_id = frame_id; + *out_wake_time_ns = wake_up_time_ns; + *out_predicted_gpu_time_ns = desired_present_time_ns; // Not quite right but close enough. + *out_predicted_display_time_ns = predicted_display_time_ns; + *out_predicted_display_period_ns = interval_ns; + + return XRT_SUCCESS; +} + +static xrt_result_t +compositor_mark_frame(struct xrt_compositor *xc, + int64_t frame_id, + enum xrt_compositor_frame_point point, + uint64_t when_ns) +{ + COMP_TRACE_MARKER(); + + struct comp_compositor *c = comp_compositor(xc); + + COMP_SPEW(c, "MARK_FRAME %i", point); + + switch (point) { + case XRT_COMPOSITOR_FRAME_POINT_WOKE: + comp_target_mark_wake_up(c->target, frame_id, when_ns); + return XRT_SUCCESS; + default: assert(false); + } + return XRT_ERROR_VULKAN; } static xrt_result_t compositor_wait_frame(struct xrt_compositor *xc, int64_t *out_frame_id, - uint64_t *predicted_display_time, - uint64_t *predicted_display_period) + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns) { + COMP_TRACE_MARKER(); + struct comp_compositor *c = comp_compositor(xc); - // A little bit easier to read. - int64_t interval_ns = (int64_t)c->settings.nominal_frame_interval_ns; + int64_t frame_id = -1; + uint64_t wake_up_time_ns = 0; + uint64_t predicted_gpu_time_ns = 0; - int64_t now_ns = os_monotonic_get_ns(); + xrt_comp_predict_frame( // + xc, // + &frame_id, // + &wake_up_time_ns, // + &predicted_gpu_time_ns, // + out_predicted_display_time_ns, // + out_predicted_display_period_ns); // - COMP_SPEW(c, "WAIT_FRAME at %8.3fms", ns_to_ms(now_ns)); - - if (c->last_next_display_time == 0) { - // First frame, we'll just assume we will display immediately - - *predicted_display_period = interval_ns; - c->last_next_display_time = now_ns + interval_ns; - *predicted_display_time = c->last_next_display_time; - *out_frame_id = c->last_next_display_time; - - COMP_SPEW(c, - "WAIT_FRAME Finished at %8.3fms, predicted display " - "time %8.3fms, period %8.3fms", - ns_to_ms(now_ns), ns_to_ms(*predicted_display_time), ns_to_ms(*predicted_display_period)); - - return XRT_SUCCESS; + uint64_t now_ns = os_monotonic_get_ns(); + if (now_ns < wake_up_time_ns) { + uint32_t delay = (uint32_t)(wake_up_time_ns - now_ns); + os_precise_sleeper_nanosleep(&c->sleeper, delay); } - // First estimate of next display time. - while (1) { + now_ns = os_monotonic_get_ns(); - int64_t render_time_ns = c->expected_app_duration_ns + c->frame_overhead_ns; - int64_t swap_interval = ceilf((float)render_time_ns / interval_ns); - int64_t render_interval_ns = swap_interval * interval_ns; - int64_t next_display_time = c->last_next_display_time + render_interval_ns; - /*! - * @todo adjust next_display_time to be a multiple of - * interval_ns from c->last_frame_time_ns - */ + xrt_comp_mark_frame(xc, frame_id, XRT_COMPOSITOR_FRAME_POINT_WOKE, now_ns); - while ((next_display_time - render_time_ns) < now_ns) { - // we can't unblock in the past - next_display_time += render_interval_ns; - } - if (compositor_wait_vsync_or_time(c, (next_display_time - render_time_ns))) { - // True return val means we actually waited for the - // deadline. - *predicted_display_period = next_display_time - c->last_next_display_time; - *predicted_display_time = next_display_time; - *out_frame_id = c->last_next_display_time; + *out_frame_id = frame_id; - c->last_next_display_time = next_display_time; - - COMP_SPEW(c, - "WAIT_FRAME Finished at %8.3fms, predicted " - "display time %8.3fms, period %8.3fms", - ns_to_ms(now_ns), ns_to_ms(*predicted_display_time), - ns_to_ms(*predicted_display_period)); - - return XRT_SUCCESS; - } - } + return XRT_SUCCESS; } static xrt_result_t @@ -258,22 +259,26 @@ compositor_add_frame_timing(struct comp_compositor *c) for (int i = 0; i < NUM_FRAME_TIMINGS; i++) { uint64_t frametime_ns = c->compositor_frame_times.times_ns[i + 1] - c->compositor_frame_times.times_ns[i]; - float frametime_s = frametime_ns * 1. / 1000. * 1. / 1000. * 1. / 1000.; + float frametime_s = frametime_ns * 1.f / 1000.f * 1.f / 1000.f * 1.f / 1000.f; total_s += frametime_s; } float avg_frametime_s = total_s / ((float)NUM_FRAME_TIMINGS); - c->compositor_frame_times.fps = 1. / avg_frametime_s; + c->compositor_frame_times.fps = 1.f / avg_frametime_s; } c->compositor_frame_times.times_ns[c->compositor_frame_times.index] = os_monotonic_get_ns(); uint64_t diff = c->compositor_frame_times.times_ns[c->compositor_frame_times.index] - c->compositor_frame_times.times_ns[last_index]; - c->compositor_frame_times.timings_ms[c->compositor_frame_times.index] = (float)diff * 1. / 1000. * 1. / 1000.; + c->compositor_frame_times.timings_ms[c->compositor_frame_times.index] = + (float)diff * 1.f / 1000.f * 1.f / 1000.f; } static xrt_result_t -compositor_layer_begin(struct xrt_compositor *xc, int64_t frame_id, enum xrt_blend_mode env_blend_mode) +compositor_layer_begin(struct xrt_compositor *xc, + int64_t frame_id, + uint64_t display_time_ns, + enum xrt_blend_mode env_blend_mode) { struct comp_compositor *c = comp_compositor(xc); @@ -401,24 +406,9 @@ compositor_layer_equirect2(struct xrt_compositor *xc, return do_single(xc, xdev, xsc, data); } -static xrt_result_t -compositor_layer_commit(struct xrt_compositor *xc, int64_t frame_id, xrt_graphics_sync_handle_t sync_handle) +static void +do_graphics_layers(struct comp_compositor *c) { - struct comp_compositor *c = comp_compositor(xc); - - COMP_SPEW(c, "LAYER_COMMIT at %8.3fms", ts_ms()); - -#ifdef XRT_GRAPHICS_SYNC_HANDLE_IS_FD - // Need to consume this handle. - if (xrt_graphics_sync_handle_is_valid(sync_handle)) { - close(sync_handle); - sync_handle = XRT_GRAPHICS_SYNC_HANDLE_INVALID; - } -#else -#error "Not yet implemented for this platform" -#endif - - // Always zero for now. uint32_t slot_id = 0; uint32_t num_layers = c->slots[slot_id].num_layers; @@ -492,6 +482,22 @@ compositor_layer_commit(struct xrt_compositor *xc, int64_t frame_id, xrt_graphic assert(false); } } +} + +static xrt_result_t +compositor_layer_commit(struct xrt_compositor *xc, int64_t frame_id, xrt_graphics_sync_handle_t sync_handle) +{ + COMP_TRACE_MARKER(); + + struct comp_compositor *c = comp_compositor(xc); + + COMP_SPEW(c, "LAYER_COMMIT at %8.3fms", ts_ms()); + + u_graphics_sync_unref(&sync_handle); + + if (!c->settings.use_compute) { + do_graphics_layers(c); + } comp_renderer_draw(c->r); @@ -501,13 +507,12 @@ compositor_layer_commit(struct xrt_compositor *xc, int64_t frame_id, xrt_graphic c->last_frame_time_ns = os_monotonic_get_ns(); c->app_profiling.last_end = c->last_frame_time_ns; - //! @todo do a time-weighted average or something. - c->expected_app_duration_ns = c->app_profiling.last_end - c->app_profiling.last_begin; COMP_SPEW(c, "LAYER_COMMIT finished drawing at %8.3fms", ns_to_ms(c->last_frame_time_ns)); // Now is a good point to garbage collect. comp_compositor_garbage_collect(c); + return XRT_SUCCESS; } @@ -551,61 +556,19 @@ static void compositor_destroy(struct xrt_compositor *xc) { struct comp_compositor *c = comp_compositor(xc); - - COMP_DEBUG(c, "COMP_DESTROY"); - - assert(c->compositor_created); - - c->compositor_created = false; -} - - -/* - * - * System compositor functions. - * - */ - -static xrt_result_t -system_compositor_create_native_compositor(struct xrt_system_compositor *xsc, - const struct xrt_session_info *xsi, - struct xrt_compositor_native **out_xcn) -{ - struct comp_compositor *c = container_of(xsc, struct comp_compositor, system); - - COMP_DEBUG(c, "SYSCOMP_CREATE_NATIVE_COMPOSITOR"); - - if (c->compositor_created) { - return XRT_ERROR_MULTI_SESSION_NOT_IMPLEMENTED; - } - - c->compositor_created = true; - c->state = COMP_STATE_PREPARED; - *out_xcn = &c->base; - - return XRT_SUCCESS; -} - -static void -system_compositor_destroy(struct xrt_system_compositor *xsc) -{ - struct comp_compositor *c = container_of(xsc, struct comp_compositor, system); struct vk_bundle *vk = &c->vk; - COMP_DEBUG(c, "SYSCOMP_DESTROY"); + COMP_DEBUG(c, "COMP_DESTROY"); // Make sure we don't have anything to destroy. comp_compositor_garbage_collect(c); - if (c->r) { - comp_renderer_destroy(c->r); - c->r = NULL; - } + comp_renderer_destroy(&c->r); comp_resources_close(c, &c->nr); // As long as vk_bundle is valid it's safe to call this function. - comp_shaders_close(&c->vk, &c->shaders); + comp_shaders_close(vk, &c->shaders); // Does NULL checking. comp_target_destroy(&c->target); @@ -620,8 +583,7 @@ system_compositor_destroy(struct xrt_system_compositor *xsc) vk->device = VK_NULL_HANDLE; } - os_mutex_destroy(&vk->queue_mutex); - os_mutex_destroy(&vk->cmd_pool_mutex); + vk_deinit_mutex(vk); if (vk->instance != VK_NULL_HANDLE) { vk->vkDestroyInstance(vk->instance, NULL); @@ -632,6 +594,8 @@ system_compositor_destroy(struct xrt_system_compositor *xsc) free(c->compositor_frame_times.debug_var); } + os_precise_sleeper_deinit(&c->sleeper); + u_threading_stack_fini(&c->threading.destroy_swapchains); free(c); @@ -690,29 +654,23 @@ compositor_check_and_prepare_xdev(struct comp_compositor *c, struct xrt_device * * */ -#define GET_DEV_PROC(c, name) (PFN_##name) c->vk.vkGetDeviceProcAddr(c->vk.device, #name); -#define GET_INS_PROC(c, name) (PFN_##name) c->vk.vkGetInstanceProcAddr(c->vk.instance, #name); -#define GET_DEV_PROC(c, name) (PFN_##name) c->vk.vkGetDeviceProcAddr(c->vk.device, #name); - // NOLINTNEXTLINE // don't remove the forward decl. VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance, const char *pName); -static VkResult -find_get_instance_proc_addr(struct comp_compositor *c) -{ - //! @todo Do any library loading here. - return vk_get_loader_functions(&c->vk, vkGetInstanceProcAddr); -} - // If any of these lists are updated, please also update the appropriate column // in `vulkan-extensions.md` -#define COMP_INSTANCE_EXTENSIONS_COMMON \ - VK_KHR_EXTERNAL_FENCE_CAPABILITIES_EXTENSION_NAME, VK_KHR_EXTERNAL_MEMORY_CAPABILITIES_EXTENSION_NAME, \ - VK_KHR_EXTERNAL_SEMAPHORE_CAPABILITIES_EXTENSION_NAME, \ - VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME, VK_KHR_SURFACE_EXTENSION_NAME +// clang-format off +#define COMP_INSTANCE_EXTENSIONS_COMMON \ + VK_EXT_DEBUG_REPORT_EXTENSION_NAME, \ + VK_KHR_EXTERNAL_FENCE_CAPABILITIES_EXTENSION_NAME, \ + VK_KHR_EXTERNAL_MEMORY_CAPABILITIES_EXTENSION_NAME, \ + VK_KHR_EXTERNAL_SEMAPHORE_CAPABILITIES_EXTENSION_NAME, \ + VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME, \ + VK_KHR_SURFACE_EXTENSION_NAME \ +// clang-format on static const char *instance_extensions_none[] = {COMP_INSTANCE_EXTENSIONS_COMMON}; @@ -721,8 +679,20 @@ static const char *instance_extensions_xcb[] = {COMP_INSTANCE_EXTENSIONS_COMMON, #endif #ifdef VK_USE_PLATFORM_WAYLAND_KHR -static const char *instance_extensions_wayland[] = {COMP_INSTANCE_EXTENSIONS_COMMON, - VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME}; +static const char *instance_extensions_wayland[] = { + COMP_INSTANCE_EXTENSIONS_COMMON, + VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME, +}; + +static const char *instance_extensions_direct_wayland[] = { + COMP_INSTANCE_EXTENSIONS_COMMON, + VK_KHR_DISPLAY_EXTENSION_NAME, + VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME, + VK_EXT_DIRECT_MODE_DISPLAY_EXTENSION_NAME, +#ifdef VK_EXT_acquire_drm_display + VK_EXT_ACQUIRE_DRM_DISPLAY_EXTENSION_NAME, +#endif +}; #endif #ifdef VK_USE_PLATFORM_XLIB_XRANDR_EXT @@ -791,6 +761,10 @@ static const char *required_device_extensions[] = { static const char *optional_device_extensions[] = { VK_GOOGLE_DISPLAY_TIMING_EXTENSION_NAME, + VK_EXT_GLOBAL_PRIORITY_EXTENSION_NAME, +#ifdef VK_EXT_robustness2 + VK_EXT_ROBUSTNESS_2_EXTENSION_NAME, +#endif }; @@ -803,6 +777,11 @@ select_instances_extensions(struct comp_compositor *c, const char ***out_exts, u *out_num = ARRAY_SIZE(instance_extensions_none); break; #ifdef VK_USE_PLATFORM_WAYLAND_KHR + case WINDOW_DIRECT_WAYLAND: + *out_exts = instance_extensions_direct_wayland; + *out_num = ARRAY_SIZE(instance_extensions_direct_wayland); + break; + case WINDOW_WAYLAND: *out_exts = instance_extensions_wayland; *out_num = ARRAY_SIZE(instance_extensions_wayland); @@ -848,6 +827,7 @@ select_instances_extensions(struct comp_compositor *c, const char ***out_exts, u static VkResult create_instance(struct comp_compositor *c) { + struct vk_bundle *vk = &c->vk; const char **instance_extensions; uint32_t num_extensions; VkResult ret; @@ -861,7 +841,7 @@ create_instance(struct comp_compositor *c) ret = select_instances_extensions(c, &instance_extensions, &num_extensions); if (ret != VK_SUCCESS) { - COMP_ERROR(c, "Failed to select instance extensions: %s", vk_result_string(ret)); + CVK_ERROR(c, "select_instances_extensions", "Failed to select instance extensions.", ret); return ret; } @@ -872,16 +852,15 @@ create_instance(struct comp_compositor *c) .ppEnabledExtensionNames = instance_extensions, }; - ret = c->vk.vkCreateInstance(&instance_info, NULL, &c->vk.instance); + ret = vk->vkCreateInstance(&instance_info, NULL, &vk->instance); if (ret != VK_SUCCESS) { - COMP_ERROR(c, "vkCreateInstance: %s\n", vk_result_string(ret)); - COMP_ERROR(c, "Failed to create Vulkan instance"); + CVK_ERROR(c, "vkCreateInstance", "Failed to create Vulkan instance", ret); return ret; } - ret = vk_get_instance_functions(&c->vk); + ret = vk_get_instance_functions(vk); if (ret != VK_SUCCESS) { - COMP_ERROR(c, "Failed to get Vulkan instance functions: %s", vk_result_string(ret)); + CVK_ERROR(c, "vk_get_instance_functions", "Failed to get Vulkan instance functions.", ret); return ret; } @@ -901,7 +880,7 @@ get_device_uuid(struct vk_bundle *vk, struct comp_compositor *c, int gpu_index, ret = vk->vkEnumeratePhysicalDevices(vk->instance, &gpu_count, phys); if (ret != VK_SUCCESS) { - COMP_ERROR(c, "Failed to enumerate physical devices!"); + CVK_ERROR(c, "vkEnumeratePhysicalDevices", "Failed to enumerate physical devices.", ret); return false; } vk->vkGetPhysicalDeviceProperties2(phys[gpu_index], &pdp2); @@ -913,41 +892,84 @@ get_device_uuid(struct vk_bundle *vk, struct comp_compositor *c, int gpu_index, static bool compositor_init_vulkan(struct comp_compositor *c) { - + struct vk_bundle *vk = &c->vk; VkResult ret; - c->vk.ll = c->settings.log_level; + vk->ll = c->settings.log_level; - ret = find_get_instance_proc_addr(c); + //! @todo Do any library loading here. + ret = vk_get_loader_functions(vk, vkGetInstanceProcAddr); if (ret != VK_SUCCESS) { + CVK_ERROR(c, "vk_get_loader_functions", "Failed to get VkInstance get process address.", ret); return false; } ret = create_instance(c); if (ret != VK_SUCCESS) { + // Error already reported. return false; } - ret = vk_create_device(&c->vk, c->settings.selected_gpu_index, required_device_extensions, - ARRAY_SIZE(required_device_extensions), optional_device_extensions, - ARRAY_SIZE(optional_device_extensions)); + const char *prio_strs[3] = { + "realtime", + "high", + "normal", + }; - if (os_mutex_init(&c->vk.queue_mutex) != 0) { - return false; - } - - if (os_mutex_init(&c->vk.cmd_pool_mutex) != 0) { + VkQueueGlobalPriorityEXT prios[3] = { + VK_QUEUE_GLOBAL_PRIORITY_REALTIME_EXT, // This is the one we really want. + VK_QUEUE_GLOBAL_PRIORITY_HIGH_EXT, // Probably not as good but something. + VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_EXT, // Default fallback. + }; + + bool use_compute = c->settings.use_compute; + + struct vk_device_features device_features = { + .shader_storage_image_write_without_format = true, + .null_descriptor = use_compute, + }; + + // No other way then to try to see if realtime is available. + for (size_t i = 0; i < ARRAY_SIZE(prios); i++) { + ret = vk_create_device( // + vk, // + c->settings.selected_gpu_index, // + use_compute, // compute_only + prios[i], // global_priority + required_device_extensions, // + ARRAY_SIZE(required_device_extensions), // + optional_device_extensions, // + ARRAY_SIZE(optional_device_extensions), // + &device_features); // optional_device_features + + // All ok! + if (ret == VK_SUCCESS) { + COMP_INFO(c, "Created device and %s queue with %s priority.", + use_compute ? "compute" : "graphics", prio_strs[i]); + break; + } + + // Try a lower priority. + if (ret == VK_ERROR_NOT_PERMITTED_EXT) { + continue; + } + + // Some other error! + CVK_ERROR(c, "vk_create_device", "Failed to create Vulkan device.", ret); return false; } + ret = vk_init_mutex(vk); if (ret != VK_SUCCESS) { + CVK_ERROR(c, "vk_init_mutex", "Failed to init mutex.", ret); return false; } - c->settings.selected_gpu_index = c->vk.physical_device_index; + + c->settings.selected_gpu_index = vk->physical_device_index; // store physical device UUID for compositor in settings if (c->settings.selected_gpu_index >= 0) { - if (get_device_uuid(&c->vk, c, c->settings.selected_gpu_index, c->settings.selected_gpu_deviceUUID)) { + if (get_device_uuid(vk, c, c->settings.selected_gpu_index, c->settings.selected_gpu_deviceUUID)) { char uuid_str[XRT_GPU_UUID_SIZE * 3 + 1] = {0}; for (int i = 0; i < XRT_GPU_UUID_SIZE; i++) { sprintf(uuid_str + i * 3, "%02x ", c->settings.selected_gpu_deviceUUID[i]); @@ -965,7 +987,7 @@ compositor_init_vulkan(struct comp_compositor *c) // store physical device UUID suggested to clients in settings if (c->settings.client_gpu_index >= 0) { - if (get_device_uuid(&c->vk, c, c->settings.client_gpu_index, c->settings.client_gpu_deviceUUID)) { + if (get_device_uuid(vk, c, c->settings.client_gpu_index, c->settings.client_gpu_deviceUUID)) { char uuid_str[XRT_GPU_UUID_SIZE * 3 + 1] = {0}; for (int i = 0; i < XRT_GPU_UUID_SIZE; i++) { sprintf(uuid_str + i * 3, "%02x ", c->settings.client_gpu_deviceUUID[i]); @@ -976,8 +998,13 @@ compositor_init_vulkan(struct comp_compositor *c) } } - ret = vk_init_cmd_pool(&c->vk); - return ret == VK_SUCCESS; + ret = vk_init_cmd_pool(vk); + if (ret != VK_SUCCESS) { + CVK_ERROR(c, "vk_init_cmd_pool", "Failed to init command pool.", ret); + return false; + } + + return true; } @@ -1015,6 +1042,8 @@ _match_wl_entry(const char *wl_entry, VkDisplayPropertiesKHR *disp) static bool _test_for_nvidia(struct comp_compositor *c, struct vk_bundle *vk) { + VkResult ret; + VkPhysicalDeviceProperties physical_device_properties; vk->vkGetPhysicalDeviceProperties(vk->physical_device, &physical_device_properties); @@ -1024,8 +1053,9 @@ _test_for_nvidia(struct comp_compositor *c, struct vk_bundle *vk) // get a list of attached displays uint32_t display_count; - if (vk->vkGetPhysicalDeviceDisplayPropertiesKHR(vk->physical_device, &display_count, NULL) != VK_SUCCESS) { - COMP_ERROR(c, "Failed to get vulkan display count"); + ret = vk->vkGetPhysicalDeviceDisplayPropertiesKHR(vk->physical_device, &display_count, NULL); + if (ret != VK_SUCCESS) { + CVK_ERROR(c, "vkGetPhysicalDeviceDisplayPropertiesKHR", "Failed to get vulkan display count", ret); return false; } @@ -1033,7 +1063,7 @@ _test_for_nvidia(struct comp_compositor *c, struct vk_bundle *vk) if (display_props && vk->vkGetPhysicalDeviceDisplayPropertiesKHR(vk->physical_device, &display_count, display_props) != VK_SUCCESS) { - COMP_ERROR(c, "Failed to get display properties"); + CVK_ERROR(c, "vkGetPhysicalDeviceDisplayPropertiesKHR", "Failed to get display properties", ret); free(display_props); return false; } @@ -1088,9 +1118,12 @@ compositor_check_vulkan_caps(struct comp_compositor *c) } COMP_DEBUG(c, "Checking for NVIDIA vulkan driver."); - struct vk_bundle temp_vk = {0}; - ret = vk_get_loader_functions(&temp_vk, vkGetInstanceProcAddr); + struct vk_bundle temp_vk_storage = {0}; + struct vk_bundle *temp_vk = &temp_vk_storage; + + ret = vk_get_loader_functions(temp_vk, vkGetInstanceProcAddr); if (ret != VK_SUCCESS) { + CVK_ERROR(c, "vk_get_loader_functions", "Failed to get loader functions.", ret); return false; } @@ -1103,35 +1136,44 @@ compositor_check_vulkan_caps(struct comp_compositor *c) .ppEnabledExtensionNames = extension_names, }; - ret = temp_vk.vkCreateInstance(&instance_create_info, NULL, &(temp_vk.instance)); + ret = temp_vk->vkCreateInstance(&instance_create_info, NULL, &(temp_vk->instance)); if (ret != VK_SUCCESS) { - COMP_ERROR(c, "Failed to create VkInstance: %s", vk_result_string(ret)); + CVK_ERROR(c, "vkCreateInstance", "Failed to create VkInstance.", ret); return false; } - ret = vk_get_instance_functions(&temp_vk); + ret = vk_get_instance_functions(temp_vk); if (ret != VK_SUCCESS) { - COMP_ERROR(c, "Failed to get Vulkan instance functions: %s", vk_result_string(ret)); + CVK_ERROR(c, "vk_get_instance_functions", "Failed to get Vulkan instance functions.", ret); return false; } + bool use_compute = c->settings.use_compute; + // follow same device selection logic as subsequent calls - ret = vk_create_device(&temp_vk, c->settings.selected_gpu_index, required_device_extensions, - ARRAY_SIZE(required_device_extensions), optional_device_extensions, - ARRAY_SIZE(optional_device_extensions)); + ret = vk_create_device( // + temp_vk, // + c->settings.selected_gpu_index, // + use_compute, // compute_only + VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_EXT, // global_priority + required_device_extensions, // + ARRAY_SIZE(required_device_extensions), // + optional_device_extensions, // + ARRAY_SIZE(optional_device_extensions), // + NULL); // optional_device_features if (ret != VK_SUCCESS) { - COMP_ERROR(c, "Failed to create VkDevice: %s", vk_result_string(ret)); + CVK_ERROR(c, "vk_create_device", "Failed to create VkDevice.", ret); return false; } - if (_test_for_nvidia(c, &temp_vk)) { + if (_test_for_nvidia(c, temp_vk)) { c->settings.window_type = WINDOW_DIRECT_NVIDIA; COMP_DEBUG(c, "Selecting direct NVIDIA window type!"); } - temp_vk.vkDestroyDevice(temp_vk.device, NULL); - temp_vk.vkDestroyInstance(temp_vk.instance, NULL); + temp_vk->vkDestroyDevice(temp_vk->device, NULL); + temp_vk->vkDestroyInstance(temp_vk->instance, NULL); #endif // VK_USE_PLATFORM_XLIB_XRANDR_EXT return true; @@ -1166,6 +1208,12 @@ compositor_init_window_pre_vulkan(struct comp_compositor *c) switch (c->settings.window_type) { case WINDOW_AUTO: +#if defined VK_USE_PLATFORM_WAYLAND_KHR && defined XRT_HAVE_WAYLAND_DIRECT + if (compositor_try_window(c, comp_window_direct_wayland_create(c))) { + c->settings.window_type = WINDOW_DIRECT_WAYLAND; + return true; + } +#endif #ifdef VK_USE_PLATFORM_WAYLAND_KHR if (compositor_try_window(c, comp_window_wayland_create(c))) { c->settings.window_type = WINDOW_WAYLAND; @@ -1181,9 +1229,7 @@ compositor_init_window_pre_vulkan(struct comp_compositor *c) #ifdef VK_USE_PLATFORM_XCB_KHR if (compositor_try_window(c, comp_window_xcb_create(c))) { c->settings.window_type = WINDOW_XCB; - COMP_DEBUG(c, - "Using VK_PRESENT_MODE_IMMEDIATE_KHR for " - "xcb window") + COMP_DEBUG(c, "Using VK_PRESENT_MODE_IMMEDIATE_KHR for xcb window") c->settings.present_mode = VK_PRESENT_MODE_IMMEDIATE_KHR; return true; } @@ -1216,6 +1262,13 @@ compositor_init_window_pre_vulkan(struct comp_compositor *c) compositor_try_window(c, comp_window_wayland_create(c)); #else COMP_ERROR(c, "Wayland support not compiled in!"); +#endif + break; + case WINDOW_DIRECT_WAYLAND: +#if defined VK_USE_PLATFORM_WAYLAND_KHR && defined XRT_HAVE_WAYLAND_DIRECT + compositor_try_window(c, comp_window_direct_wayland_create(c)); +#else + COMP_ERROR(c, "Wayland direct support not compiled in!"); #endif break; case WINDOW_DIRECT_RANDR: @@ -1250,6 +1303,8 @@ compositor_init_window_pre_vulkan(struct comp_compositor *c) static bool compositor_init_window_post_vulkan(struct comp_compositor *c) { + os_precise_sleeper_init(&c->sleeper); + if (c->settings.window_type == WINDOW_DIRECT_NVIDIA) { #ifdef VK_USE_PLATFORM_XLIB_XRANDR_EXT return compositor_try_window(c, comp_window_direct_nvidia_create(c)); @@ -1274,25 +1329,14 @@ compositor_init_window_post_vulkan(struct comp_compositor *c) static bool compositor_init_swapchain(struct comp_compositor *c) { - if (!comp_target_init_post_vulkan(c->target, // - c->settings.preferred.width, // - c->settings.preferred.height)) { - COMP_ERROR(c, "Window init_swapchain failed!"); - goto err_destroy; + if (comp_target_init_post_vulkan(c->target, // + c->settings.preferred.width, // + c->settings.preferred.height)) { + return true; } - comp_target_create_images( // - c->target, // - c->settings.preferred.width, // - c->settings.preferred.height, // - c->settings.color_format, // - c->settings.color_space, // - c->settings.present_mode); // + COMP_ERROR(c, "Window init_swapchain failed!"); - return true; - - // Error path. -err_destroy: comp_target_destroy(&c->target); return false; @@ -1301,7 +1345,9 @@ err_destroy: static bool compositor_init_shaders(struct comp_compositor *c) { - return comp_shaders_load(&c->vk, &c->shaders); + struct vk_bundle *vk = &c->vk; + + return comp_shaders_load(vk, &c->shaders); } static bool @@ -1318,8 +1364,10 @@ compositor_init_renderer(struct comp_compositor *c) bool comp_is_format_supported(struct comp_compositor *c, VkFormat format) { + struct vk_bundle *vk = &c->vk; VkFormatProperties prop; - c->vk.vkGetPhysicalDeviceFormatProperties(c->vk.physical_device, format, &prop); + + vk->vkGetPhysicalDeviceFormatProperties(vk->physical_device, format, &prop); // This is a fairly crude way of checking support, // but works well enough. @@ -1340,8 +1388,11 @@ xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compos c->base.base.create_swapchain = comp_swapchain_create; c->base.base.import_swapchain = comp_swapchain_import; + c->base.base.import_fence = comp_compositor_import_fence; c->base.base.begin_session = compositor_begin_session; c->base.base.end_session = compositor_end_session; + c->base.base.predict_frame = compositor_predict_frame; + c->base.base.mark_frame = compositor_mark_frame; c->base.base.wait_frame = compositor_wait_frame; c->base.base.begin_frame = compositor_begin_frame; c->base.base.discard_frame = compositor_discard_frame; @@ -1356,8 +1407,8 @@ xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compos c->base.base.layer_commit = compositor_layer_commit; c->base.base.poll_events = compositor_poll_events; c->base.base.destroy = compositor_destroy; - c->system.create_native_compositor = system_compositor_create_native_compositor; - c->system.destroy = system_compositor_destroy; + c->frame.waited.id = -1; + c->frame.rendering.id = -1; c->xdev = xdev; u_threading_stack_init(&c->threading.destroy_swapchains); @@ -1368,9 +1419,6 @@ xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compos comp_settings_init(&c->settings, xdev); c->last_frame_time_ns = os_monotonic_get_ns(); - c->frame_overhead_ns = 2000000; - //! @todo set this to an estimate that's better than 6ms - c->expected_app_duration_ns = 6000000; // Need to select window backend before creating Vulkan, then @@ -1387,8 +1435,8 @@ xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compos !compositor_init_shaders(c) || !compositor_init_swapchain(c) || !compositor_init_renderer(c)) { - COMP_DEBUG(c, "Failed to init compositor %p", (void *)c); - c->system.destroy(&c->system); + COMP_ERROR(c, "Failed to init compositor %p", (void *)c); + c->base.base.destroy(&c->base.base); return XRT_ERROR_VULKAN; } @@ -1416,12 +1464,26 @@ xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compos uint32_t formats = 0; // color formats - ADD_IF_SUPPORTED(VK_FORMAT_A2B10G10R10_UNORM_PACK32); // OGL VK - ADD_IF_SUPPORTED(VK_FORMAT_R16G16B16A16_SFLOAT); // OGL VK - ADD_IF_SUPPORTED(VK_FORMAT_R8G8B8A8_SRGB); // OGL VK - ADD_IF_SUPPORTED(VK_FORMAT_B8G8R8A8_SRGB); // VK - ADD_IF_SUPPORTED(VK_FORMAT_R8G8B8A8_UNORM); // OGL VK - ADD_IF_SUPPORTED(VK_FORMAT_B8G8R8A8_UNORM); // VK + /* + * The format VK_FORMAT_A2B10G10R10_UNORM_PACK32 is not listed since + * 10 bits are not considered enough to do linear colours without + * banding. If there was a sRGB variant of it then we would have used it + * instead but there isn't. Since it's not a popular format it's best + * not to list it rather then listing it and people falling into the + * trap. The absolute minimum is R11G11B10, but is a really weird format + * so we are not exposing it. + */ + ADD_IF_SUPPORTED(VK_FORMAT_R16G16B16A16_UNORM); // OGL VK + ADD_IF_SUPPORTED(VK_FORMAT_R16G16B16A16_SFLOAT); // OGL VK + ADD_IF_SUPPORTED(VK_FORMAT_R16G16B16_UNORM); // OGL VK - Uncommon. + ADD_IF_SUPPORTED(VK_FORMAT_R16G16B16_SFLOAT); // OGL VK - Uncommon. + ADD_IF_SUPPORTED(VK_FORMAT_R8G8B8A8_SRGB); // OGL VK + ADD_IF_SUPPORTED(VK_FORMAT_B8G8R8A8_SRGB); // VK + ADD_IF_SUPPORTED(VK_FORMAT_R8G8B8_SRGB); // OGL VK - Uncommon. + ADD_IF_SUPPORTED(VK_FORMAT_R8G8B8A8_UNORM); // OGL VK - Bad colour precision. + ADD_IF_SUPPORTED(VK_FORMAT_B8G8R8A8_UNORM); // VK - Bad colour precision. + ADD_IF_SUPPORTED(VK_FORMAT_R8G8B8_UNORM); // OGL VK - Uncommon. Bad colour precision. + ADD_IF_SUPPORTED(VK_FORMAT_B8G8R8_UNORM); // VK - Uncommon. Bad colour precision. // depth formats ADD_IF_SUPPORTED(VK_FORMAT_D16_UNORM); // OGL VK @@ -1434,7 +1496,8 @@ xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compos assert(formats <= XRT_MAX_SWAPCHAIN_FORMATS); info->num_formats = formats; - struct xrt_system_compositor_info *sys_info = &c->system.info; + struct xrt_system_compositor_info sys_info_storage; + struct xrt_system_compositor_info *sys_info = &sys_info_storage; // Required by OpenXR spec. sys_info->max_layers = 16; @@ -1477,10 +1540,11 @@ xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compos u_var_add_root(c, "Compositor", true); u_var_add_ro_f32(c, &c->compositor_frame_times.fps, "FPS (Compositor)"); + u_var_add_bool(c, &c->debug.atw_off, "Debug: ATW OFF"); struct u_var_timing *ft = U_TYPED_CALLOC(struct u_var_timing); - float target_frame_time_ms = c->settings.nominal_frame_interval_ns * 1. / 1000. * 1. / 1000.; + float target_frame_time_ms = ns_to_ms(c->settings.nominal_frame_interval_ns); uint64_t now = os_monotonic_get_ns(); for (int i = 0; i < NUM_FRAME_TIMES; i++) { @@ -1502,9 +1566,7 @@ xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compos c->state = COMP_STATE_READY; - *out_xsysc = &c->system; - - return XRT_SUCCESS; + return comp_multi_create_system_compositor(&c->base, sys_info, out_xsysc); } void diff --git a/src/xrt/compositor/main/comp_compositor.h b/src/xrt/compositor/main/comp_compositor.h index f1d9be8da..69ca7be39 100644 --- a/src/xrt/compositor/main/comp_compositor.h +++ b/src/xrt/compositor/main/comp_compositor.h @@ -139,6 +139,10 @@ enum comp_state struct comp_shaders { + VkShaderModule clear_comp; + VkShaderModule distortion_comp; + VkShaderModule distortion_timewarp_comp; + VkShaderModule mesh_vert; VkShaderModule mesh_frag; @@ -152,6 +156,17 @@ struct comp_shaders VkShaderModule layer_frag; }; +/*! + * Tracking frame state. + */ +struct comp_frame +{ + int64_t id; + uint64_t predicted_display_time_ns; + uint64_t desired_present_time_ns; + uint64_t present_slop_ns; +}; + /*! * Main compositor struct tying everything in the compositor together. * @@ -162,8 +177,6 @@ struct comp_compositor { struct xrt_compositor_native base; - struct xrt_system_compositor system; - //! Renderer helper. struct comp_renderer *r; @@ -188,6 +201,8 @@ struct comp_compositor //! State for generating the correct set of events. enum comp_state state; + struct os_precise_sleeper sleeper; + //! Triple buffered layer stacks. struct comp_layer_slot slots[3]; @@ -201,9 +216,6 @@ struct comp_compositor int64_t last_end; } app_profiling; - //! The time our compositor needs to do rendering - int64_t frame_overhead_ns; - struct { //! Current Index for times_ns. @@ -221,16 +233,11 @@ struct comp_compositor struct u_var_timing *debug_var; } compositor_frame_times; - /*! - * @brief Estimated rendering time per frame of the application. - * - * Set by the begin_frame/end_frame code. - * - * @todo make this atomic. - */ - int64_t expected_app_duration_ns; - //! The last time we provided in the results of wait_frame - int64_t last_next_display_time; + struct + { + struct comp_frame waited; + struct comp_frame rendering; + } frame; struct { @@ -239,10 +246,13 @@ struct comp_compositor } threading; - struct comp_resources nr; + struct + { + //! Temporarily disable ATW + bool atw_off; + } debug; - //! To insure only one compositor is created. - bool compositor_created; + struct comp_resources nr; }; @@ -321,6 +331,14 @@ comp_swapchain_import(struct xrt_compositor *xc, void comp_swapchain_really_destroy(struct comp_swapchain *sc); +/*! + * For importing fences, defined in comp_sync.c . + */ +xrt_result_t +comp_compositor_import_fence(struct xrt_compositor *xc, + xrt_graphics_sync_handle_t handle, + struct xrt_compositor_fence **out_xcf); + /*! * Loads all of the shaders that the compositor uses. */ @@ -328,7 +346,7 @@ bool comp_shaders_load(struct vk_bundle *vk, struct comp_shaders *s); /*! - * Loads all of the shaders that the compositor uses. + * Unload and cleanup shaders. */ void comp_shaders_close(struct vk_bundle *vk, struct comp_shaders *s); diff --git a/src/xrt/compositor/main/comp_layer.c b/src/xrt/compositor/main/comp_layer.c index 83e82fe84..0ea9278d4 100644 --- a/src/xrt/compositor/main/comp_layer.c +++ b/src/xrt/compositor/main/comp_layer.c @@ -51,19 +51,25 @@ _update_mvp_matrix(struct comp_render_layer *self, uint32_t eye, const struct xr static bool _init_ubos(struct comp_render_layer *self) { + struct vk_bundle *vk = self->vk; + VkBufferUsageFlags usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT; - VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | - VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; for (uint32_t i = 0; i < 2; i++) { math_matrix_4x4_identity(&self->transformation[i].mvp); - if (!vk_buffer_init(self->vk, sizeof(struct layer_transformation), usage, properties, - &self->transformation_ubos[i].handle, &self->transformation_ubos[i].memory)) + if (!vk_buffer_init(vk, // + sizeof(struct layer_transformation), // + usage, // + properties, // + &self->transformation_ubos[i].handle, // + &self->transformation_ubos[i].memory)) { return false; + } - VkResult res = self->vk->vkMapMemory(self->vk->device, self->transformation_ubos[i].memory, 0, - VK_WHOLE_SIZE, 0, &self->transformation_ubos[i].data); + VkResult res = vk->vkMapMemory(vk->device, self->transformation_ubos[i].memory, 0, VK_WHOLE_SIZE, 0, + &self->transformation_ubos[i].data); vk_check_error("vkMapMemory", res, false); memcpy(self->transformation_ubos[i].data, &self->transformation[i], @@ -76,16 +82,22 @@ _init_ubos(struct comp_render_layer *self) static bool _init_equirect1_ubo(struct comp_render_layer *self) { + struct vk_bundle *vk = self->vk; + VkBufferUsageFlags usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT; - VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | - VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; - if (!vk_buffer_init(self->vk, sizeof(struct layer_transformation), usage, properties, - &self->equirect1_ubo.handle, &self->equirect1_ubo.memory)) + if (!vk_buffer_init(vk, // + sizeof(struct layer_transformation), // + usage, // + properties, // + &self->equirect1_ubo.handle, // + &self->equirect1_ubo.memory)) { return false; + } - VkResult res = self->vk->vkMapMemory(self->vk->device, self->equirect1_ubo.memory, 0, VK_WHOLE_SIZE, 0, - &self->equirect1_ubo.data); + VkResult res = + vk->vkMapMemory(vk->device, self->equirect1_ubo.memory, 0, VK_WHOLE_SIZE, 0, &self->equirect1_ubo.data); vk_check_error("vkMapMemory", res, false); memcpy(self->equirect1_ubo.data, &self->equirect1_data, sizeof(struct layer_equirect1_data)); @@ -97,16 +109,22 @@ _init_equirect1_ubo(struct comp_render_layer *self) static bool _init_equirect2_ubo(struct comp_render_layer *self) { + struct vk_bundle *vk = self->vk; + VkBufferUsageFlags usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT; - VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | - VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; - if (!vk_buffer_init(self->vk, sizeof(struct layer_transformation), usage, properties, - &self->equirect2_ubo.handle, &self->equirect2_ubo.memory)) + if (!vk_buffer_init(vk, // + sizeof(struct layer_transformation), // + usage, // + properties, // + &self->equirect2_ubo.handle, // + &self->equirect2_ubo.memory)) { return false; + } - VkResult res = self->vk->vkMapMemory(self->vk->device, self->equirect2_ubo.memory, 0, VK_WHOLE_SIZE, 0, - &self->equirect2_ubo.data); + VkResult res = + vk->vkMapMemory(vk->device, self->equirect2_ubo.memory, 0, VK_WHOLE_SIZE, 0, &self->equirect2_ubo.data); vk_check_error("vkMapMemory", res, false); memcpy(self->equirect2_ubo.data, &self->equirect2_data, sizeof(struct layer_equirect2_data)); @@ -162,6 +180,8 @@ _update_descriptor(struct comp_render_layer *self, static void _update_descriptor_equirect(struct comp_render_layer *self, VkDescriptorSet set, VkBuffer buffer) { + struct vk_bundle *vk = self->vk; + VkWriteDescriptorSet *sets = (VkWriteDescriptorSet[]){ { .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, @@ -179,16 +199,19 @@ _update_descriptor_equirect(struct comp_render_layer *self, VkDescriptorSet set, }, }; - self->vk->vkUpdateDescriptorSets(self->vk->device, 1, sets, 0, NULL); + vk->vkUpdateDescriptorSets(vk->device, 1, sets, 0, NULL); } #endif void comp_layer_update_descriptors(struct comp_render_layer *self, VkSampler sampler, VkImageView image_view) { - for (uint32_t eye = 0; eye < 2; eye++) - _update_descriptor(self, self->vk, self->descriptor_sets[eye], self->transformation_ubos[eye].handle, - sampler, image_view); + struct vk_bundle *vk = self->vk; + + for (uint32_t eye = 0; eye < 2; eye++) { + _update_descriptor(self, vk, self->descriptor_sets[eye], self->transformation_ubos[eye].handle, sampler, + image_view); + } } #ifdef XRT_FEATURE_OPENXR_LAYER_EQUIRECT1 @@ -228,10 +251,12 @@ comp_layer_update_stereo_descriptors(struct comp_render_layer *self, VkImageView left_image_view, VkImageView right_image_view) { - _update_descriptor(self, self->vk, self->descriptor_sets[0], self->transformation_ubos[0].handle, left_sampler, + struct vk_bundle *vk = self->vk; + + _update_descriptor(self, vk, self->descriptor_sets[0], self->transformation_ubos[0].handle, left_sampler, left_image_view); - _update_descriptor(self, self->vk, self->descriptor_sets[1], self->transformation_ubos[1].handle, right_sampler, + _update_descriptor(self, vk, self->descriptor_sets[1], self->transformation_ubos[1].handle, right_sampler, right_image_view); } @@ -271,17 +296,15 @@ _init(struct comp_render_layer *self, }, }; - if (!vk_init_descriptor_pool(self->vk, pool_sizes, ARRAY_SIZE(pool_sizes), 3, &self->descriptor_pool)) + if (!vk_init_descriptor_pool(vk, pool_sizes, ARRAY_SIZE(pool_sizes), 3, &self->descriptor_pool)) return false; for (uint32_t eye = 0; eye < 2; eye++) - if (!vk_allocate_descriptor_sets(self->vk, self->descriptor_pool, 1, layout, - &self->descriptor_sets[eye])) + if (!vk_allocate_descriptor_sets(vk, self->descriptor_pool, 1, layout, &self->descriptor_sets[eye])) return false; #if defined(XRT_FEATURE_OPENXR_LAYER_EQUIRECT1) || defined(XRT_FEATURE_OPENXR_LAYER_EQUIRECT2) - if (!vk_allocate_descriptor_sets(self->vk, self->descriptor_pool, 1, layout_equirect, - &self->descriptor_equirect)) + if (!vk_allocate_descriptor_sets(vk, self->descriptor_pool, 1, layout_equirect, &self->descriptor_equirect)) return false; #endif return true; @@ -297,6 +320,8 @@ comp_layer_draw(struct comp_render_layer *self, const struct xrt_matrix_4x4 *vp_world, const struct xrt_matrix_4x4 *vp_eye) { + struct vk_bundle *vk = self->vk; + if (eye == 0 && (self->visibility & XRT_LAYER_EYE_VISIBILITY_LEFT_BIT) == 0) { return; } @@ -305,7 +330,7 @@ comp_layer_draw(struct comp_render_layer *self, return; } - self->vk->vkCmdBindPipeline(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + vk->vkCmdBindPipeline(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); // Is this layer viewspace or not. const struct xrt_matrix_4x4 *vp = self->view_space ? vp_eye : vp_world; @@ -329,18 +354,18 @@ comp_layer_draw(struct comp_render_layer *self, self->descriptor_equirect, }; - self->vk->vkCmdBindDescriptorSets(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline_layout, 0, 2, - sets, 0, NULL); + vk->vkCmdBindDescriptorSets(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline_layout, 0, 2, sets, 0, + NULL); } else { - self->vk->vkCmdBindDescriptorSets(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline_layout, 0, 1, - &self->descriptor_sets[eye], 0, NULL); + vk->vkCmdBindDescriptorSets(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline_layout, 0, 1, + &self->descriptor_sets[eye], 0, NULL); } VkDeviceSize offsets[1] = {0}; - self->vk->vkCmdBindVertexBuffers(cmd_buffer, 0, 1, &vertex_buffer->handle, &offsets[0]); + vk->vkCmdBindVertexBuffers(cmd_buffer, 0, 1, &vertex_buffer->handle, &offsets[0]); - self->vk->vkCmdDraw(cmd_buffer, vertex_buffer->size, 1, 0, 0); + vk->vkCmdDraw(cmd_buffer, vertex_buffer->size, 1, 0, 0); } // clang-format off @@ -447,11 +472,17 @@ _init_cylinder_vertex_buffer(struct comp_render_layer *self) VkBufferUsageFlags usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; - if (!vk_buffer_init(vk, sizeof(float) * ARRAY_SIZE(cylinder_vertices), usage, properties, - &self->cylinder.vertex_buffer.handle, &self->cylinder.vertex_buffer.memory)) + if (!vk_buffer_init(vk, // + sizeof(float) * ARRAY_SIZE(cylinder_vertices), // + usage, // + properties, // + &self->cylinder.vertex_buffer.handle, // + &self->cylinder.vertex_buffer.memory)) { return false; + } self->cylinder.vertex_buffer.size = CYLINDER_VERTICES; + return true; } @@ -468,8 +499,9 @@ comp_layer_create(struct vk_bundle *vk, VkDescriptorSetLayout *layout, VkDescrip _init(q, vk, layout, layout_equirect); - if (!_init_cylinder_vertex_buffer(q)) + if (!_init_cylinder_vertex_buffer(q)) { return NULL; + } return q; } @@ -477,18 +509,21 @@ comp_layer_create(struct vk_bundle *vk, VkDescriptorSetLayout *layout, VkDescrip void comp_layer_destroy(struct comp_render_layer *self) { - for (uint32_t eye = 0; eye < 2; eye++) - vk_buffer_destroy(&self->transformation_ubos[eye], self->vk); + struct vk_bundle *vk = self->vk; + + for (uint32_t eye = 0; eye < 2; eye++) { + vk_buffer_destroy(&self->transformation_ubos[eye], vk); + } #ifdef XRT_FEATURE_OPENXR_LAYER_EQUIRECT1 - vk_buffer_destroy(&self->equirect1_ubo, self->vk); + vk_buffer_destroy(&self->equirect1_ubo, vk); #endif #ifdef XRT_FEATURE_OPENXR_LAYER_EQUIRECT2 - vk_buffer_destroy(&self->equirect2_ubo, self->vk); + vk_buffer_destroy(&self->equirect2_ubo, vk); #endif - self->vk->vkDestroyDescriptorPool(self->vk->device, self->descriptor_pool, NULL); + vk->vkDestroyDescriptorPool(vk->device, self->descriptor_pool, NULL); - vk_buffer_destroy(&self->cylinder.vertex_buffer, self->vk); + vk_buffer_destroy(&self->cylinder.vertex_buffer, vk); free(self); } diff --git a/src/xrt/compositor/main/comp_layer_renderer.c b/src/xrt/compositor/main/comp_layer_renderer.c index b3202f743..d8a0a6321 100644 --- a/src/xrt/compositor/main/comp_layer_renderer.c +++ b/src/xrt/compositor/main/comp_layer_renderer.c @@ -22,10 +22,16 @@ struct comp_layer_vertex float uv[2]; }; -static const VkClearColorValue background_color = { - .float32 = {0.3f, 0.3f, 0.3f, 1.0f}, +static const VkClearColorValue background_color_idle = { + .float32 = {0.1f, 0.1f, 0.1f, 1.0f}, }; +static const VkClearColorValue background_color_active = { + .float32 = {0.0f, 0.0f, 0.0f, 1.0f}, +}; + + + static bool _init_render_pass(struct vk_bundle *vk, VkFormat format, @@ -308,7 +314,7 @@ _init_graphics_pipeline(struct comp_layer_renderer *self, VK_DYNAMIC_STATE_SCISSOR, }, }, - .subpass = VK_NULL_HANDLE, + .subpass = 0, }; VkResult res; @@ -571,8 +577,13 @@ _render_pass_begin(struct vk_bundle *vk, } static void -_render_stereo(struct comp_layer_renderer *self, struct vk_bundle *vk, VkCommandBuffer cmd_buffer) +_render_stereo(struct comp_layer_renderer *self, + struct vk_bundle *vk, + VkCommandBuffer cmd_buffer, + const VkClearColorValue *color) { + COMP_TRACE_MARKER(); + VkViewport viewport = { 0.0f, 0.0f, self->extent.width, self->extent.height, 0.0f, 1.0f, }; @@ -584,8 +595,8 @@ _render_stereo(struct comp_layer_renderer *self, struct vk_bundle *vk, VkCommand vk->vkCmdSetScissor(cmd_buffer, 0, 1, &scissor); for (uint32_t eye = 0; eye < 2; eye++) { - _render_pass_begin(vk, self->render_pass, self->extent, background_color, - self->framebuffers[eye].handle, cmd_buffer); + _render_pass_begin(vk, self->render_pass, self->extent, *color, self->framebuffers[eye].handle, + cmd_buffer); _render_eye(self, eye, cmd_buffer, self->pipeline_layout); @@ -596,14 +607,19 @@ _render_stereo(struct comp_layer_renderer *self, struct vk_bundle *vk, VkCommand void comp_layer_renderer_draw(struct comp_layer_renderer *self) { + COMP_TRACE_MARKER(); + struct vk_bundle *vk = self->vk; VkCommandBuffer cmd_buffer; if (vk_init_cmd_buffer(vk, &cmd_buffer) != VK_SUCCESS) return; - os_mutex_lock(&vk->cmd_pool_mutex); - _render_stereo(self, vk, cmd_buffer); + if (self->num_layers == 0) { + _render_stereo(self, vk, cmd_buffer, &background_color_idle); + } else { + _render_stereo(self, vk, cmd_buffer, &background_color_active); + } os_mutex_unlock(&vk->cmd_pool_mutex); VkResult res = vk_submit_cmd_buffer(vk, cmd_buffer); @@ -622,8 +638,15 @@ _destroy_framebuffer(struct comp_layer_renderer *self, uint32_t i) } void -comp_layer_renderer_destroy(struct comp_layer_renderer *self) +comp_layer_renderer_destroy(struct comp_layer_renderer **ptr_clr) { + if (ptr_clr == NULL) { + return; + } + struct comp_layer_renderer *self = *ptr_clr; + if (self == NULL) { + return; + } struct vk_bundle *vk = self->vk; if (vk->device == VK_NULL_HANDLE) @@ -654,6 +677,8 @@ comp_layer_renderer_destroy(struct comp_layer_renderer *self) vk_buffer_destroy(&self->vertex_buffer, vk); vk->vkDestroyPipelineCache(vk->device, self->pipeline_cache, NULL); + free(self); + *ptr_clr = NULL; } void diff --git a/src/xrt/compositor/main/comp_layer_renderer.h b/src/xrt/compositor/main/comp_layer_renderer.h index 5fbf4cd12..2b5f3f77f 100644 --- a/src/xrt/compositor/main/comp_layer_renderer.h +++ b/src/xrt/compositor/main/comp_layer_renderer.h @@ -1,4 +1,4 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -7,16 +7,15 @@ * @ingroup comp_main */ +#pragma once + +#include "comp_layer.h" + /*! * Holds associated vulkan objects and state to render quads. * * @ingroup comp_main */ - -#pragma once - -#include "comp_layer.h" - struct comp_layer_renderer { struct vk_bundle *vk; @@ -63,26 +62,77 @@ struct comp_layer_renderer uint32_t texture_binding; }; +/*! + * Create a layer renderer. + * + * @public @memberof comp_layer_renderer + */ struct comp_layer_renderer * comp_layer_renderer_create(struct vk_bundle *vk, struct comp_shaders *s, VkExtent2D extent, VkFormat format); +/*! + * Destroy the layer renderer and set the pointer to NULL. + * + * @public @memberof comp_layer_renderer + */ void -comp_layer_renderer_destroy(struct comp_layer_renderer *self); +comp_layer_renderer_destroy(struct comp_layer_renderer **ptr_clr); +/*! + * Perform draw calls for the layers. + * + * @param self Self pointer. + * + * @public @memberof comp_layer_renderer + */ void comp_layer_renderer_draw(struct comp_layer_renderer *self); +/*! + * Update the internal members derived from the field of view. + * + * @param self Self pointer. + * @param fov Field of view data + * @param eye Eye index: 0 or 1 + * + * @public @memberof comp_layer_renderer + */ void comp_layer_renderer_set_fov(struct comp_layer_renderer *self, const struct xrt_fov *fov, uint32_t eye); +/*! + * Update the internal members derived from the eye and world poses. + * + * @param self Self pointer. + * @param eye_pose Pose of eye in view + * @param world_pose Pose of eye in world + * @param eye Eye index: 0 or 1 + * + * @public @memberof comp_layer_renderer + */ void comp_layer_renderer_set_pose(struct comp_layer_renderer *self, const struct xrt_pose *eye_pose, const struct xrt_pose *world_pose, uint32_t eye); +/*! + * Allocate the array comp_layer_renderer::layers with the given number of elements. + * + * @param self Self pointer. + * @param num_layers The number of layers to support + * + * @public @memberof comp_layer_renderer + */ void comp_layer_renderer_allocate_layers(struct comp_layer_renderer *self, uint32_t num_layers); +/*! + * De-initialize and free comp_layer_renderer::layers array. + * + * @param self Self pointer. + * + * @public @memberof comp_layer_renderer + */ void comp_layer_renderer_destroy_layers(struct comp_layer_renderer *self); diff --git a/src/xrt/compositor/main/comp_renderer.c b/src/xrt/compositor/main/comp_renderer.c index 0d1043ce8..d7451c7ae 100644 --- a/src/xrt/compositor/main/comp_renderer.c +++ b/src/xrt/compositor/main/comp_renderer.c @@ -1,22 +1,28 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Compositor rendering code. * @author Lubosz Sarnecki * @author Jakob Bornecrantz + * @author Ryan Pavlik * @ingroup comp_main */ #include "xrt/xrt_compositor.h" +#include "os/os_time.h" + #include "math/m_space.h" #include "util/u_misc.h" +#include "util/u_trace_marker.h" #include "util/u_distortion_mesh.h" #include "main/comp_layer_renderer.h" #include "math/m_api.h" +#include "math/m_vec3.h" +#include "math/m_matrix_4x4_f64.h" #include #include @@ -38,9 +44,13 @@ */ struct comp_renderer { - uint32_t current_buffer; + //! @name Durable members + //! @brief These don't require the images to be created and don't depend on it. + //! @{ - VkQueue queue; + //! The compositor we were created by + struct comp_compositor *c; + struct comp_settings *settings; struct { @@ -48,81 +58,43 @@ struct comp_renderer VkSemaphore render_complete; } semaphores; + VkQueue queue; + //! @} + + //! @name Image-dependent members + //! @{ + + //! Index of the current buffer/image + int32_t acquired_buffer; + + //! Which buffer was last submitted and has a fence pending. + int32_t fenced_buffer; + + /*! + * Array of "renderings" equal in size to the number of comp_target images. + */ struct comp_rendering *rrs; + + /*! + * Array of fences equal in size to the number of comp_target images. + */ VkFence *fences; + + /*! + * The number of renderings/fences we've created: set from comp_target when we use that data. + */ uint32_t num_buffers; - struct comp_compositor *c; - struct comp_settings *settings; - + /*! + * @brief The layer renderer, which actually knows how to composite layers. + * + * Depends on the target extents. + */ struct comp_layer_renderer *lr; + //! @} }; -/* - * - * Pre declare functions. - * - */ - -static void -renderer_create(struct comp_renderer *r, struct comp_compositor *c); - -static void -renderer_init(struct comp_renderer *r); - -static void -renderer_submit_queue(struct comp_renderer *r); - -static void -renderer_build_renderings(struct comp_renderer *r); - -static void -renderer_allocate_renderings(struct comp_renderer *r); - -static void -renderer_close_renderings(struct comp_renderer *r); - -static void -renderer_init_semaphores(struct comp_renderer *r); - -static void -renderer_resize(struct comp_renderer *r); - -static void -renderer_acquire_swapchain_image(struct comp_renderer *r); - -static void -renderer_present_swapchain_image(struct comp_renderer *r); - -static void -renderer_destroy(struct comp_renderer *r); - - -/* - * - * Interface functions. - * - */ - -struct comp_renderer * -comp_renderer_create(struct comp_compositor *c) -{ - struct comp_renderer *r = U_TYPED_CALLOC(struct comp_renderer); - - renderer_create(r, c); - renderer_init(r); - - return r; -} - -void -comp_renderer_destroy(struct comp_renderer *r) -{ - renderer_destroy(r); - free(r); -} - /* * * Functions. @@ -130,54 +102,96 @@ comp_renderer_destroy(struct comp_renderer *r) */ static void -renderer_create(struct comp_renderer *r, struct comp_compositor *c) +renderer_wait_gpu_idle(struct comp_renderer *r) { - r->c = c; - r->settings = &c->settings; + COMP_TRACE_MARKER(); + struct vk_bundle *vk = &r->c->vk; - r->current_buffer = 0; - r->queue = VK_NULL_HANDLE; - r->semaphores.present_complete = VK_NULL_HANDLE; - r->semaphores.render_complete = VK_NULL_HANDLE; - - r->rrs = NULL; + os_mutex_lock(&vk->queue_mutex); + vk->vkDeviceWaitIdle(vk->device); + os_mutex_unlock(&vk->queue_mutex); } static void -renderer_submit_queue(struct comp_renderer *r) +renderer_init_semaphores(struct comp_renderer *r) { struct vk_bundle *vk = &r->c->vk; VkResult ret; - VkPipelineStageFlags stage_flags[1] = { - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VkSemaphoreCreateInfo info = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, }; - ret = vk->vkWaitForFences(vk->device, 1, &r->fences[r->current_buffer], VK_TRUE, UINT64_MAX); - if (ret != VK_SUCCESS) - COMP_ERROR(r->c, "vkWaitForFences: %s", vk_result_string(ret)); - - ret = vk->vkResetFences(vk->device, 1, &r->fences[r->current_buffer]); - if (ret != VK_SUCCESS) - COMP_ERROR(r->c, "vkResetFences: %s", vk_result_string(ret)); - - VkSubmitInfo comp_submit_info = { - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, - .waitSemaphoreCount = 1, - .pWaitSemaphores = &r->semaphores.present_complete, - .pWaitDstStageMask = stage_flags, - .commandBufferCount = 1, - .pCommandBuffers = &r->rrs[r->current_buffer].cmd, - .signalSemaphoreCount = 1, - .pSignalSemaphores = &r->semaphores.render_complete, - }; - - ret = vk_locked_submit(vk, r->queue, 1, &comp_submit_info, r->fences[r->current_buffer]); + ret = vk->vkCreateSemaphore(vk->device, &info, NULL, &r->semaphores.present_complete); if (ret != VK_SUCCESS) { - COMP_ERROR(r->c, "vkQueueSubmit: %s", vk_result_string(ret)); + COMP_ERROR(r->c, "vkCreateSemaphore: %s", vk_result_string(ret)); + } + + ret = vk->vkCreateSemaphore(vk->device, &info, NULL, &r->semaphores.render_complete); + if (ret != VK_SUCCESS) { + COMP_ERROR(r->c, "vkCreateSemaphore: %s", vk_result_string(ret)); } } +static void +calc_viewport_data(struct comp_renderer *r, + struct comp_viewport_data *out_l_viewport_data, + struct comp_viewport_data *out_r_viewport_data) +{ + struct comp_compositor *c = r->c; + + bool pre_rotate = false; + if (r->c->target->surface_transform & VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR || + r->c->target->surface_transform & VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR) { + COMP_SPEW(c, "Swapping width and height, since we are pre rotating"); + pre_rotate = true; + } + + float w = pre_rotate ? r->c->xdev->hmd->screens[0].h_pixels : r->c->xdev->hmd->screens[0].w_pixels; + float h = pre_rotate ? r->c->xdev->hmd->screens[0].w_pixels : r->c->xdev->hmd->screens[0].h_pixels; + + float scale_x = (float)r->c->target->width / w; + float scale_y = (float)r->c->target->height / h; + + struct xrt_view *l_v = &r->c->xdev->hmd->views[0]; + struct xrt_view *r_v = &r->c->xdev->hmd->views[1]; + + struct comp_viewport_data l_viewport_data; + struct comp_viewport_data r_viewport_data; + + if (pre_rotate) { + l_viewport_data = (struct comp_viewport_data){ + .x = (uint32_t)(l_v->viewport.y_pixels * scale_x), + .y = (uint32_t)(l_v->viewport.x_pixels * scale_y), + .w = (uint32_t)(l_v->viewport.h_pixels * scale_x), + .h = (uint32_t)(l_v->viewport.w_pixels * scale_y), + }; + r_viewport_data = (struct comp_viewport_data){ + .x = (uint32_t)(r_v->viewport.y_pixels * scale_x), + .y = (uint32_t)(r_v->viewport.x_pixels * scale_y), + .w = (uint32_t)(r_v->viewport.h_pixels * scale_x), + .h = (uint32_t)(r_v->viewport.w_pixels * scale_y), + }; + } else { + l_viewport_data = (struct comp_viewport_data){ + .x = (uint32_t)(l_v->viewport.x_pixels * scale_x), + .y = (uint32_t)(l_v->viewport.y_pixels * scale_y), + .w = (uint32_t)(l_v->viewport.w_pixels * scale_x), + .h = (uint32_t)(l_v->viewport.h_pixels * scale_y), + }; + r_viewport_data = (struct comp_viewport_data){ + .x = (uint32_t)(r_v->viewport.x_pixels * scale_x), + .y = (uint32_t)(r_v->viewport.y_pixels * scale_y), + .w = (uint32_t)(r_v->viewport.w_pixels * scale_x), + .h = (uint32_t)(r_v->viewport.h_pixels * scale_y), + }; + } + + *out_l_viewport_data = l_viewport_data; + *out_r_viewport_data = r_viewport_data; +} + +//! @pre comp_target_has_images(r->c->target) static void renderer_build_rendering(struct comp_renderer *r, struct comp_rendering *rr, uint32_t index) { @@ -192,39 +206,25 @@ renderer_build_rendering(struct comp_renderer *r, struct comp_rendering *rr, uin bool pre_rotate = false; if (r->c->target->surface_transform & VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR || r->c->target->surface_transform & VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR) { - COMP_DEBUG(c, - "Swapping width and height," - "since we are pre rotating"); + COMP_DEBUG(c, "Swapping width and height, since we are pre rotating"); pre_rotate = true; } + struct comp_viewport_data l_viewport_data; + struct comp_viewport_data r_viewport_data; - float w = pre_rotate ? r->c->xdev->hmd->screens[0].h_pixels : r->c->xdev->hmd->screens[0].w_pixels; - float h = pre_rotate ? r->c->xdev->hmd->screens[0].w_pixels : r->c->xdev->hmd->screens[0].h_pixels; - - float scale_x = (float)r->c->target->width / w; - float scale_y = (float)r->c->target->height / h; + calc_viewport_data(r, &l_viewport_data, &r_viewport_data); struct xrt_view *l_v = &r->c->xdev->hmd->views[0]; + struct xrt_view *r_v = &r->c->xdev->hmd->views[1]; + struct comp_mesh_ubo_data l_data = { + .vertex_rot = l_v->rot, + }; - struct comp_viewport_data l_viewport_data; - - if (pre_rotate) { - l_viewport_data = (struct comp_viewport_data){ - .x = (uint32_t)(l_v->viewport.y_pixels * scale_x), - .y = (uint32_t)(l_v->viewport.x_pixels * scale_y), - .w = (uint32_t)(l_v->viewport.h_pixels * scale_x), - .h = (uint32_t)(l_v->viewport.w_pixels * scale_y), - }; - } else { - l_viewport_data = (struct comp_viewport_data){ - .x = (uint32_t)(l_v->viewport.x_pixels * scale_x), - .y = (uint32_t)(l_v->viewport.y_pixels * scale_y), - .w = (uint32_t)(l_v->viewport.w_pixels * scale_x), - .h = (uint32_t)(l_v->viewport.h_pixels * scale_y), - }; - } + struct comp_mesh_ubo_data r_data = { + .vertex_rot = r_v->rot, + }; const struct xrt_matrix_2x2 rotation_90_cw = {{ .vecs = @@ -234,43 +234,9 @@ renderer_build_rendering(struct comp_renderer *r, struct comp_rendering *rr, uin }, }}; - - struct comp_mesh_ubo_data l_data = { - .rot = l_v->rot, - .flip_y = false, - }; - if (pre_rotate) { - math_matrix_2x2_multiply(&l_v->rot, &rotation_90_cw, &l_data.rot); - } - - struct xrt_view *r_v = &r->c->xdev->hmd->views[1]; - - struct comp_viewport_data r_viewport_data; - - if (pre_rotate) { - r_viewport_data = (struct comp_viewport_data){ - .x = (uint32_t)(r_v->viewport.y_pixels * scale_x), - .y = (uint32_t)(r_v->viewport.x_pixels * scale_y), - .w = (uint32_t)(r_v->viewport.h_pixels * scale_x), - .h = (uint32_t)(r_v->viewport.w_pixels * scale_y), - }; - } else { - r_viewport_data = (struct comp_viewport_data){ - .x = (uint32_t)(r_v->viewport.x_pixels * scale_x), - .y = (uint32_t)(r_v->viewport.y_pixels * scale_y), - .w = (uint32_t)(r_v->viewport.w_pixels * scale_x), - .h = (uint32_t)(r_v->viewport.h_pixels * scale_y), - }; - } - - struct comp_mesh_ubo_data r_data = { - .rot = r_v->rot, - .flip_y = false, - }; - - if (pre_rotate) { - math_matrix_2x2_multiply(&r_v->rot, &rotation_90_cw, &r_data.rot); + math_matrix_2x2_multiply(&l_v->rot, &rotation_90_cw, &l_data.vertex_rot); + math_matrix_2x2_multiply(&r_v->rot, &rotation_90_cw, &r_data.vertex_rot); } /* @@ -326,38 +292,281 @@ renderer_build_rendering(struct comp_renderer *r, struct comp_rendering *rr, uin comp_draw_end_target(rr); } +/*! + * @pre comp_target_has_images(r->c->target) + * Update r->num_buffers before calling. + */ static void -renderer_build_renderings(struct comp_renderer *r) +renderer_create_renderings_and_fences(struct comp_renderer *r) { - for (uint32_t i = 0; i < r->num_buffers; ++i) { - renderer_build_rendering(r, &r->rrs[i], i); + assert(r->rrs == NULL); + assert(r->fences == NULL); + if (r->num_buffers == 0) { + COMP_ERROR(r->c, "Requested 0 command buffers."); + return; } -} -static void -renderer_create_fences(struct comp_renderer *r) -{ - r->fences = U_TYPED_ARRAY_CALLOC(VkFence, r->num_buffers); + COMP_DEBUG(r->c, "Allocating %d Command Buffers.", r->num_buffers); struct vk_bundle *vk = &r->c->vk; + bool use_compute = r->settings->use_compute; + if (!use_compute) { + r->rrs = U_TYPED_ARRAY_CALLOC(struct comp_rendering, r->num_buffers); + + for (uint32_t i = 0; i < r->num_buffers; ++i) { + renderer_build_rendering(r, &r->rrs[i], i); + } + } + + r->fences = U_TYPED_ARRAY_CALLOC(VkFence, r->num_buffers); + for (uint32_t i = 0; i < r->num_buffers; i++) { - VkResult ret = vk->vkCreateFence(vk->device, - &(VkFenceCreateInfo){.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, - .flags = VK_FENCE_CREATE_SIGNALED_BIT}, - NULL, &r->fences[i]); + VkFenceCreateInfo fence_info = { + .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, + .flags = VK_FENCE_CREATE_SIGNALED_BIT, + }; + + VkResult ret = vk->vkCreateFence( // + vk->device, // + &fence_info, // + NULL, // + &r->fences[i]); // if (ret != VK_SUCCESS) { COMP_ERROR(r->c, "vkCreateFence: %s", vk_result_string(ret)); } } } +static void +renderer_close_renderings_and_fences(struct comp_renderer *r) +{ + struct vk_bundle *vk = &r->c->vk; + // Renderings + if (r->num_buffers > 0 && r->rrs != NULL) { + for (uint32_t i = 0; i < r->num_buffers; i++) { + comp_rendering_close(&r->rrs[i]); + } + + free(r->rrs); + r->rrs = NULL; + } + + // Fences + if (r->num_buffers > 0 && r->fences != NULL) { + for (uint32_t i = 0; i < r->num_buffers; i++) { + vk->vkDestroyFence(vk->device, r->fences[i], NULL); + r->fences[i] = VK_NULL_HANDLE; + } + free(r->fences); + r->fences = NULL; + } + + r->num_buffers = 0; + r->acquired_buffer = -1; + r->fenced_buffer = -1; +} + +//! @pre comp_target_check_ready(r->c->target) +static void +renderer_create_layer_renderer(struct comp_renderer *r) +{ + struct vk_bundle *vk = &r->c->vk; + + assert(comp_target_check_ready(r->c->target)); + + uint32_t num_layers = 0; + if (r->lr != NULL) { + // if we already had one, re-populate it after recreation. + num_layers = r->lr->num_layers; + comp_layer_renderer_destroy(&r->lr); + } + + VkExtent2D extent; + if (r->c->target->surface_transform & VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR || + r->c->target->surface_transform & VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR) { + // Swapping width and height, since we are pre rotating + extent = (VkExtent2D){ + .width = r->c->xdev->hmd->screens[0].h_pixels, + .height = r->c->xdev->hmd->screens[0].w_pixels, + }; + } else { + extent = (VkExtent2D){ + .width = r->c->xdev->hmd->screens[0].w_pixels, + .height = r->c->xdev->hmd->screens[0].h_pixels, + }; + } + + r->lr = comp_layer_renderer_create(vk, &r->c->shaders, extent, VK_FORMAT_B8G8R8A8_SRGB); + if (num_layers != 0) { + comp_layer_renderer_allocate_layers(r->lr, num_layers); + } +} + +/*! + * @brief Ensure that target images and renderings are created, if possible. + * + * @param r Self pointer + * @param force_recreate If true, will tear down and re-create images and renderings, e.g. for a resize + * + * @returns true if images and renderings are ready and created. + * + * @private @memberof comp_renderer + * @ingroup comp_main + */ +static bool +renderer_ensure_images_and_renderings(struct comp_renderer *r, bool force_recreate) +{ + struct comp_compositor *c = r->c; + struct comp_target *target = c->target; + + if (!comp_target_check_ready(target)) { + // Not ready, so can't render anything. + return false; + } + + // We will create images if we don't have any images or if we were told to recreate them. + bool create = force_recreate || !comp_target_has_images(target) || (r->num_buffers == 0); + if (!create) { + return true; + } + + COMP_DEBUG(c, "Creating iamges and renderings (force_recreate: %s).", force_recreate ? "true" : "false"); + + /* + * This makes sure that any pending command buffer has completed + * and all resources referred by it can now be manipulated. This + * make sure that validation doesn't complain. This is done + * during resize so isn't time critical. + */ + renderer_wait_gpu_idle(r); + + // Make we sure we destroy all dependent things before creating new images. + renderer_close_renderings_and_fences(r); + + VkImageUsageFlags image_usage = 0; + if (r->settings->use_compute) { + image_usage = VK_IMAGE_USAGE_STORAGE_BIT; + } else { + image_usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + } + + comp_target_create_images( // + r->c->target, // + r->c->settings.preferred.width, // + r->c->settings.preferred.height, // + r->settings->color_format, // + r->settings->color_space, // + image_usage, // + r->settings->present_mode); // + + r->num_buffers = r->c->target->num_images; + + renderer_create_layer_renderer(r); + renderer_create_renderings_and_fences(r); + + assert(r->num_buffers != 0); + + return true; +} + +//! Create renderer and initialize non-image-dependent members +static void +renderer_create(struct comp_renderer *r, struct comp_compositor *c) +{ + r->c = c; + r->settings = &c->settings; + + r->acquired_buffer = -1; + r->fenced_buffer = -1; + r->queue = VK_NULL_HANDLE; + r->semaphores.present_complete = VK_NULL_HANDLE; + r->semaphores.render_complete = VK_NULL_HANDLE; + r->rrs = NULL; + + struct vk_bundle *vk = &r->c->vk; + + vk->vkGetDeviceQueue(vk->device, vk->queue_family_index, 0, &r->queue); + renderer_init_semaphores(r); + + // Try to early-allocate these, in case we can. + renderer_ensure_images_and_renderings(r, false); +} + +static void +renderer_wait_for_last_fence(struct comp_renderer *r) +{ + COMP_TRACE_MARKER(); + + if (r->fenced_buffer < 0) { + return; + } + + struct vk_bundle *vk = &r->c->vk; + VkResult ret; + + ret = vk->vkWaitForFences(vk->device, 1, &r->fences[r->fenced_buffer], VK_TRUE, UINT64_MAX); + if (ret != VK_SUCCESS) { + COMP_ERROR(r->c, "vkWaitForFences: %s", vk_result_string(ret)); + } + + r->fenced_buffer = -1; +} + +static void +renderer_submit_queue(struct comp_renderer *r, VkCommandBuffer cmd) +{ + COMP_TRACE_MARKER(); + + struct vk_bundle *vk = &r->c->vk; + VkResult ret; + + VkPipelineStageFlags stage_flags[1] = { + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + }; + + // Wait for the last fence, if any. + renderer_wait_for_last_fence(r); + assert(r->fenced_buffer < 0); + + assert(r->acquired_buffer >= 0); + ret = vk->vkResetFences(vk->device, 1, &r->fences[r->acquired_buffer]); + if (ret != VK_SUCCESS) { + COMP_ERROR(r->c, "vkResetFences: %s", vk_result_string(ret)); + } + + VkSubmitInfo comp_submit_info = { + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, + .waitSemaphoreCount = 1, + .pWaitSemaphores = &r->semaphores.present_complete, + .pWaitDstStageMask = stage_flags, + .commandBufferCount = 1, + .pCommandBuffers = &cmd, + .signalSemaphoreCount = 1, + .pSignalSemaphores = &r->semaphores.render_complete, + }; + + ret = vk_locked_submit(vk, r->queue, 1, &comp_submit_info, r->fences[r->acquired_buffer]); + if (ret != VK_SUCCESS) { + COMP_ERROR(r->c, "vkQueueSubmit: %s", vk_result_string(ret)); + } + + // This buffer now have a pending fence. + r->fenced_buffer = r->acquired_buffer; +} + static void renderer_get_view_projection(struct comp_renderer *r) { + COMP_TRACE_MARKER(); + struct xrt_space_relation relation; - xrt_device_get_tracked_pose(r->c->xdev, XRT_INPUT_GENERIC_HEAD_POSE, r->c->last_next_display_time, &relation); + xrt_device_get_tracked_pose( // + r->c->xdev, // + XRT_INPUT_GENERIC_HEAD_POSE, // + r->c->frame.rendering.predicted_display_time_ns, // + &relation); // struct xrt_vec3 eye_relation = { 0.063000f, /* TODO: get actual ipd_meters */ @@ -365,10 +574,7 @@ renderer_get_view_projection(struct comp_renderer *r) 0.0f, }; - struct xrt_pose base_space_pose = { - .position = (struct xrt_vec3){0, 0, 0}, - .orientation = (struct xrt_quat){0, 0, 0, 1}, - }; + struct xrt_pose base_space_pose = XRT_POSE_IDENTITY; for (uint32_t i = 0; i < 2; i++) { struct xrt_fov fov = r->c->xdev->hmd->views[i].fov; @@ -390,48 +596,308 @@ renderer_get_view_projection(struct comp_renderer *r) } static void -renderer_init(struct comp_renderer *r) +renderer_acquire_swapchain_image(struct comp_renderer *r) +{ + COMP_TRACE_MARKER(); + + uint32_t buffer_index = 0; + VkResult ret; + + assert(r->acquired_buffer < 0); + + if (!renderer_ensure_images_and_renderings(r, false)) { + // Not ready yet. + return; + } + ret = comp_target_acquire(r->c->target, r->semaphores.present_complete, &buffer_index); + + if ((ret == VK_ERROR_OUT_OF_DATE_KHR) || (ret == VK_SUBOPTIMAL_KHR)) { + COMP_DEBUG(r->c, "Received %s.", vk_result_string(ret)); + + if (!renderer_ensure_images_and_renderings(r, true)) { + // Failed on force recreate. + COMP_ERROR(r->c, + "renderer_acquire_swapchain_image: comp_target_acquire was out of date, force " + "re-create image and renderings failed. Probably the target disappeared."); + return; + } + + /* Acquire image again to silence validation error */ + ret = comp_target_acquire(r->c->target, r->semaphores.present_complete, &buffer_index); + if (ret != VK_SUCCESS) { + COMP_ERROR(r->c, "comp_target_acquire: %s", vk_result_string(ret)); + } + } else if (ret != VK_SUCCESS) { + COMP_ERROR(r->c, "comp_target_acquire: %s", vk_result_string(ret)); + } + + r->acquired_buffer = buffer_index; +} + +static void +renderer_resize(struct comp_renderer *r) +{ + if (!comp_target_check_ready(r->c->target)) { + // Can't create images right now. + // Just close any existing renderings. + renderer_close_renderings_and_fences(r); + return; + } + + renderer_ensure_images_and_renderings(r, true); // Force recreate. + + return; +} + +static void +renderer_present_swapchain_image(struct comp_renderer *r, uint64_t desired_present_time_ns, uint64_t present_slop_ns) +{ + COMP_TRACE_MARKER(); + + VkResult ret; + + ret = comp_target_present( // + r->c->target, // + r->queue, // + r->acquired_buffer, // + r->semaphores.render_complete, // + desired_present_time_ns, // + present_slop_ns); // + r->acquired_buffer = -1; + + if (ret == VK_ERROR_OUT_OF_DATE_KHR || ret == VK_SUBOPTIMAL_KHR) { + renderer_resize(r); + return; + } + if (ret != VK_SUCCESS) { + COMP_ERROR(r->c, "vk_swapchain_present: %s", vk_result_string(ret)); + } +} + +static void +renderer_destroy(struct comp_renderer *r) { struct vk_bundle *vk = &r->c->vk; - vk->vkGetDeviceQueue(vk->device, r->c->vk.queue_family_index, 0, &r->queue); - renderer_init_semaphores(r); - assert(r->c->target->num_images > 0); + // Command buffers + renderer_close_renderings_and_fences(r); - r->num_buffers = r->c->target->num_images; - - renderer_create_fences(r); - - VkExtent2D extent; - if (r->c->target->surface_transform & VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR || - r->c->target->surface_transform & VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR) { - // Swapping width and height, since we are pre rotating - extent = (VkExtent2D){ - .width = r->c->xdev->hmd->screens[0].h_pixels, - .height = r->c->xdev->hmd->screens[0].w_pixels, - }; - } else { - extent = (VkExtent2D){ - .width = r->c->xdev->hmd->screens[0].w_pixels, - .height = r->c->xdev->hmd->screens[0].h_pixels, - }; + // Semaphores + if (r->semaphores.present_complete != VK_NULL_HANDLE) { + vk->vkDestroySemaphore(vk->device, r->semaphores.present_complete, NULL); + r->semaphores.present_complete = VK_NULL_HANDLE; + } + if (r->semaphores.render_complete != VK_NULL_HANDLE) { + vk->vkDestroySemaphore(vk->device, r->semaphores.render_complete, NULL); + r->semaphores.render_complete = VK_NULL_HANDLE; } - r->lr = comp_layer_renderer_create(vk, &r->c->shaders, extent, VK_FORMAT_B8G8R8A8_SRGB); - - renderer_allocate_renderings(r); - renderer_build_renderings(r); + comp_layer_renderer_destroy(&(r->lr)); } -VkImageView -get_image_view(struct comp_swapchain_image *image, enum xrt_layer_composition_flags flags, uint32_t array_index) +static VkImageView +get_image_view(const struct comp_swapchain_image *image, enum xrt_layer_composition_flags flags, uint32_t array_index) { if (flags & XRT_LAYER_COMPOSITION_BLEND_TEXTURE_SOURCE_ALPHA_BIT) { return image->views.alpha[array_index]; } + return image->views.no_alpha[array_index]; } +static void +dispatch_graphics(struct comp_renderer *r) +{ + COMP_TRACE_MARKER(); + + struct comp_compositor *c = r->c; + struct comp_target *ct = c->target; + + comp_target_mark_submit(ct, c->frame.rendering.id, os_monotonic_get_ns()); + + renderer_get_view_projection(r); + comp_layer_renderer_draw(r->lr); + + comp_target_update_timings(ct); + + renderer_submit_queue(r, r->rrs[r->acquired_buffer].cmd); +} + + +/* + * + * Compute + * + */ + +static void +get_view_poses(struct comp_renderer *r, struct xrt_pose out_results[2]) +{ + COMP_TRACE_MARKER(); + + struct xrt_space_relation relation; + + xrt_device_get_tracked_pose( // + r->c->xdev, // + XRT_INPUT_GENERIC_HEAD_POSE, // + r->c->frame.rendering.predicted_display_time_ns, // + &relation); // + + struct xrt_vec3 eye_relation = { + 0.063000f, /* TODO: get actual ipd_meters */ + 0.0f, + 0.0f, + }; + + for (uint32_t i = 0; i < 2; i++) { + struct xrt_fov fov = r->c->xdev->hmd->views[i].fov; + + comp_layer_renderer_set_fov(r->lr, &fov, i); + + struct xrt_pose eye_pose = XRT_POSE_IDENTITY; + xrt_device_get_view_pose(r->c->xdev, &eye_relation, i, &eye_pose); + + struct xrt_space_relation result = {0}; + struct xrt_space_graph xsg = {0}; + m_space_graph_add_pose_if_not_identity(&xsg, &eye_pose); + m_space_graph_add_relation(&xsg, &relation); + m_space_graph_resolve(&xsg, &result); + + out_results[i] = result.pose; + } +} + +static void +do_projection_layers(struct comp_renderer *r, + struct comp_rendering_compute *crc, + const struct comp_layer *layer, + const struct xrt_layer_projection_view_data *lvd, + const struct xrt_layer_projection_view_data *rvd) +{ + const struct xrt_layer_data *data = &layer->data; + uint32_t left_array_index = lvd->sub.array_index; + uint32_t right_array_index = rvd->sub.array_index; + const struct comp_swapchain_image *left = &layer->scs[0]->images[lvd->sub.image_index]; + const struct comp_swapchain_image *right = &layer->scs[1]->images[rvd->sub.image_index]; + + struct comp_viewport_data views[2]; + calc_viewport_data(r, &views[0], &views[1]); + + VkImage target_image = r->c->target->images[r->acquired_buffer].handle; + VkImageView target_image_view = r->c->target->images[r->acquired_buffer].view; + + struct xrt_pose new_view_poses[2]; + get_view_poses(r, new_view_poses); + + VkSampler src_samplers[2] = { + left->sampler, + right->sampler, + }; + + VkImageView src_image_views[2] = { + get_image_view(left, data->flags, left_array_index), + get_image_view(right, data->flags, right_array_index), + }; + + struct xrt_normalized_rect src_norm_rects[2] = {lvd->sub.norm_rect, rvd->sub.norm_rect}; + if (data->flip_y) { + src_norm_rects[0].h = -src_norm_rects[0].h; + src_norm_rects[0].y = 1 + src_norm_rects[0].y; + src_norm_rects[1].h = -src_norm_rects[1].h; + src_norm_rects[1].y = 1 + src_norm_rects[1].y; + } + + struct xrt_pose src_poses[2] = { + lvd->pose, + rvd->pose, + }; + + struct xrt_fov src_fovs[2] = { + lvd->fov, + rvd->fov, + }; + + if (crc->c->debug.atw_off) { + comp_rendering_compute_projection( // + crc, // + src_samplers, // + src_image_views, // + src_norm_rects, // + target_image, // + target_image_view, // + views); // + } else { + comp_rendering_compute_projection_timewarp( // + crc, // + src_samplers, // + src_image_views, // + src_norm_rects, // + src_poses, // + src_fovs, // + new_view_poses, // + target_image, // + target_image_view, // + views); // + } +} + +static void +dispatch_compute(struct comp_renderer *r, struct comp_rendering_compute *crc) +{ + COMP_TRACE_MARKER(); + + struct comp_compositor *c = r->c; + struct comp_target *ct = c->target; + + comp_rendering_compute_init(c, &c->nr, crc); + comp_rendering_compute_begin(crc); + + struct comp_viewport_data views[2]; + calc_viewport_data(r, &views[0], &views[1]); + + VkImage target_image = r->c->target->images[r->acquired_buffer].handle; + VkImageView target_image_view = r->c->target->images[r->acquired_buffer].view; + + uint32_t slot_id = 0; + uint32_t num_layers = c->slots[slot_id].num_layers; + if (num_layers > 0 && c->slots[slot_id].layers[0].data.type == XRT_LAYER_STEREO_PROJECTION) { + int i = 0; + const struct comp_layer *layer = &c->slots[slot_id].layers[i]; + const struct xrt_layer_stereo_projection_data *stereo = &layer->data.stereo; + const struct xrt_layer_projection_view_data *lvd = &stereo->l; + const struct xrt_layer_projection_view_data *rvd = &stereo->r; + + do_projection_layers(r, crc, layer, lvd, rvd); + } else if (num_layers > 0 && c->slots[slot_id].layers[0].data.type == XRT_LAYER_STEREO_PROJECTION_DEPTH) { + int i = 0; + const struct comp_layer *layer = &c->slots[slot_id].layers[i]; + const struct xrt_layer_stereo_projection_depth_data *stereo = &layer->data.stereo_depth; + const struct xrt_layer_projection_view_data *lvd = &stereo->l; + const struct xrt_layer_projection_view_data *rvd = &stereo->r; + + do_projection_layers(r, crc, layer, lvd, rvd); + } else { + comp_rendering_compute_clear( // + crc, // + target_image, // + target_image_view, // + views); // + } + + comp_rendering_compute_end(crc); + + comp_target_mark_submit(ct, c->frame.rendering.id, os_monotonic_get_ns()); + + renderer_submit_queue(r, crc->cmd); +} + + +/* + * + * Interface functions. + * + */ + void comp_renderer_set_quad_layer(struct comp_renderer *r, uint32_t layer, @@ -577,6 +1043,7 @@ comp_renderer_set_equirect1_layer(struct comp_renderer *r, } } #endif + #ifdef XRT_FEATURE_OPENXR_LAYER_EQUIRECT2 void comp_renderer_set_equirect2_layer(struct comp_renderer *r, @@ -614,14 +1081,51 @@ comp_renderer_set_equirect2_layer(struct comp_renderer *r, void comp_renderer_draw(struct comp_renderer *r) { - renderer_get_view_projection(r); - comp_layer_renderer_draw(r->lr); + COMP_TRACE_MARKER(); - comp_target_flush(r->c->target); + struct comp_target *ct = r->c->target; + struct comp_compositor *c = r->c; - renderer_acquire_swapchain_image(r); - renderer_submit_queue(r); - renderer_present_swapchain_image(r); + + assert(c->frame.rendering.id == -1); + + c->frame.rendering = c->frame.waited; + c->frame.waited.id = -1; + + comp_target_mark_begin(ct, c->frame.rendering.id, os_monotonic_get_ns()); + + // Are we ready to render? No - skip rendering. + if (!comp_target_check_ready(r->c->target)) { + // Need to emulate rendering for the timing. + //! @todo This should be discard. + comp_target_mark_submit(ct, c->frame.rendering.id, os_monotonic_get_ns()); + return; + } + + comp_target_flush(ct); + + comp_target_update_timings(ct); + + if (r->acquired_buffer < 0) { + // Ensures that renderings are created. + renderer_acquire_swapchain_image(r); + } + + comp_target_update_timings(ct); + + bool use_compute = r->settings->use_compute; + struct comp_rendering_compute crc = {0}; + if (use_compute) { + dispatch_compute(r, &crc); + } else { + dispatch_graphics(r); + } + + renderer_present_swapchain_image(r, c->frame.rendering.desired_present_time_ns, + c->frame.rendering.present_slop_ns); + + // Clear the frame. + c->frame.rendering.id = -1; /* * This fixes a lot of validation issues as it makes sure that the @@ -630,168 +1134,61 @@ comp_renderer_draw(struct comp_renderer *r) * * This is done after a swap so isn't time critical. */ - os_mutex_lock(&r->c->vk.queue_mutex); - r->c->vk.vkDeviceWaitIdle(r->c->vk.device); - os_mutex_unlock(&r->c->vk.queue_mutex); -} + renderer_wait_gpu_idle(r); -static void -renderer_allocate_renderings(struct comp_renderer *r) -{ - if (r->num_buffers == 0) { - COMP_ERROR(r->c, "Requested 0 command buffers."); - return; + if (use_compute) { + comp_rendering_compute_close(&crc); } - COMP_DEBUG(r->c, "Allocating %d Command Buffers.", r->num_buffers); - - if (r->rrs != NULL) { - free(r->rrs); - } - - r->rrs = U_TYPED_ARRAY_CALLOC(struct comp_rendering, r->num_buffers); -} - -static void -renderer_close_renderings(struct comp_renderer *r) -{ - for (uint32_t i = 0; i < r->num_buffers; i++) { - comp_rendering_close(&r->rrs[i]); - } - - free(r->rrs); - r->rrs = NULL; -} - -static void -renderer_init_semaphores(struct comp_renderer *r) -{ - struct vk_bundle *vk = &r->c->vk; - VkResult ret; - - VkSemaphoreCreateInfo info = { - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, - }; - - ret = vk->vkCreateSemaphore(vk->device, &info, NULL, &r->semaphores.present_complete); - if (ret != VK_SUCCESS) { - COMP_ERROR(r->c, "vkCreateSemaphore: %s", vk_result_string(ret)); - } - - ret = vk->vkCreateSemaphore(vk->device, &info, NULL, &r->semaphores.render_complete); - if (ret != VK_SUCCESS) { - COMP_ERROR(r->c, "vkCreateSemaphore: %s", vk_result_string(ret)); - } -} - -static void -renderer_resize(struct comp_renderer *r) -{ - struct vk_bundle *vk = &r->c->vk; - /* - * This makes sure that any pending command buffer has completed - * and all resources referred by it can now be manipulated. This - * make sure that validation doesn't complain. This is done - * during resize so isn't time critical. + * For direct mode this makes us wait until the last frame has been + * actually shown to the user, this avoids us missing that we have + * missed a frame and miss-predicting the next frame. */ - os_mutex_lock(&vk->queue_mutex); - vk->vkDeviceWaitIdle(vk->device); - os_mutex_unlock(&vk->queue_mutex); + renderer_acquire_swapchain_image(r); - comp_target_create_images( // - r->c->target, // - r->c->target->width, // - r->c->target->height, // - r->settings->color_format, // - r->settings->color_space, // - r->settings->present_mode); // - - renderer_close_renderings(r); - - r->num_buffers = r->c->target->num_images; - - renderer_allocate_renderings(r); - renderer_build_renderings(r); -} - -static void -renderer_acquire_swapchain_image(struct comp_renderer *r) -{ - VkResult ret; - - ret = comp_target_acquire(r->c->target, r->semaphores.present_complete, &r->current_buffer); - - if ((ret == VK_ERROR_OUT_OF_DATE_KHR) || (ret == VK_SUBOPTIMAL_KHR)) { - COMP_DEBUG(r->c, "Received %s.", vk_result_string(ret)); - renderer_resize(r); - - /* Acquire image again to silence validation error */ - ret = comp_target_acquire(r->c->target, r->semaphores.present_complete, &r->current_buffer); - if (ret != VK_SUCCESS) { - COMP_ERROR(r->c, "vk_swapchain_acquire_next_image: %s", vk_result_string(ret)); - } - } else if (ret != VK_SUCCESS) { - COMP_ERROR(r->c, "vk_swapchain_acquire_next_image: %s", vk_result_string(ret)); - } -} - -static void -renderer_present_swapchain_image(struct comp_renderer *r) -{ - VkResult ret; - - ret = comp_target_present(r->c->target, r->queue, r->current_buffer, r->semaphores.render_complete); - if (ret == VK_ERROR_OUT_OF_DATE_KHR) { - renderer_resize(r); - return; - } - if (ret != VK_SUCCESS) { - COMP_ERROR(r->c, "vk_swapchain_present: %s", vk_result_string(ret)); - } -} - -static void -renderer_destroy(struct comp_renderer *r) -{ - struct vk_bundle *vk = &r->c->vk; - - // Fences - for (uint32_t i = 0; i < r->num_buffers; i++) - vk->vkDestroyFence(vk->device, r->fences[i], NULL); - free(r->fences); - - // Command buffers - renderer_close_renderings(r); - if (r->rrs != NULL) { - free(r->rrs); - } - - r->num_buffers = 0; - - // Semaphores - if (r->semaphores.present_complete != VK_NULL_HANDLE) { - vk->vkDestroySemaphore(vk->device, r->semaphores.present_complete, NULL); - r->semaphores.present_complete = VK_NULL_HANDLE; - } - if (r->semaphores.render_complete != VK_NULL_HANDLE) { - vk->vkDestroySemaphore(vk->device, r->semaphores.render_complete, NULL); - r->semaphores.render_complete = VK_NULL_HANDLE; - } - - comp_layer_renderer_destroy(r->lr); - - free(r->lr); + comp_target_update_timings(ct); } void comp_renderer_allocate_layers(struct comp_renderer *self, uint32_t num_layers) { + COMP_TRACE_MARKER(); + comp_layer_renderer_allocate_layers(self->lr, num_layers); } void comp_renderer_destroy_layers(struct comp_renderer *self) { + COMP_TRACE_MARKER(); + comp_layer_renderer_destroy_layers(self->lr); } + +struct comp_renderer * +comp_renderer_create(struct comp_compositor *c) +{ + struct comp_renderer *r = U_TYPED_CALLOC(struct comp_renderer); + + renderer_create(r, c); + + return r; +} + +void +comp_renderer_destroy(struct comp_renderer **ptr_r) +{ + if (ptr_r == NULL) { + return; + } + + struct comp_renderer *r = *ptr_r; + if (r == NULL) { + return; + } + + renderer_destroy(r); + free(r); + *ptr_r = NULL; +} diff --git a/src/xrt/compositor/main/comp_renderer.h b/src/xrt/compositor/main/comp_renderer.h index ff78c3d91..c1edaaf5a 100644 --- a/src/xrt/compositor/main/comp_renderer.h +++ b/src/xrt/compositor/main/comp_renderer.h @@ -1,4 +1,4 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -19,41 +19,46 @@ extern "C" { struct comp_compositor; -struct comp_renderer; struct comp_swapchain_image; +/*! + * @brief Renderer used by compositor. + */ +struct comp_renderer; /*! * Called by the main compositor code to create the renderer. * + * @public @memberof comp_renderer + * @see comp_compositor * @ingroup comp_main */ struct comp_renderer * comp_renderer_create(struct comp_compositor *c); -/*! - * Reset renderer as input has changed. - * - * @ingroup comp_main - */ -void -comp_renderer_reset(struct comp_renderer *r); - /*! * Clean up and free the renderer. * + * Does null checking and sets to null after freeing. + * + * @public @memberof comp_renderer * @ingroup comp_main */ void -comp_renderer_destroy(struct comp_renderer *r); +comp_renderer_destroy(struct comp_renderer **ptr_r); /*! * Render frame. * + * @public @memberof comp_renderer * @ingroup comp_main */ void comp_renderer_draw(struct comp_renderer *r); +/*! + * @public @memberof comp_renderer + * @ingroup comp_main + */ void comp_renderer_set_projection_layer(struct comp_renderer *r, uint32_t layer, @@ -61,12 +66,20 @@ comp_renderer_set_projection_layer(struct comp_renderer *r, struct comp_swapchain_image *right_image, struct xrt_layer_data *data); +/*! + * @public @memberof comp_renderer + * @ingroup comp_main + */ void comp_renderer_set_quad_layer(struct comp_renderer *r, uint32_t layer, struct comp_swapchain_image *image, struct xrt_layer_data *data); +/*! + * @public @memberof comp_renderer + * @ingroup comp_main + */ void comp_renderer_set_cylinder_layer(struct comp_renderer *r, uint32_t layer, @@ -74,13 +87,22 @@ comp_renderer_set_cylinder_layer(struct comp_renderer *r, struct xrt_layer_data *data); #ifdef XRT_FEATURE_OPENXR_LAYER_EQUIRECT1 +/*! + * @public @memberof comp_renderer + * @ingroup comp_main + */ void comp_renderer_set_equirect1_layer(struct comp_renderer *r, uint32_t layer, struct comp_swapchain_image *image, struct xrt_layer_data *data); #endif + #ifdef XRT_FEATURE_OPENXR_LAYER_EQUIRECT2 +/*! + * @public @memberof comp_renderer + * @ingroup comp_main + */ void comp_renderer_set_equirect2_layer(struct comp_renderer *r, uint32_t layer, @@ -88,9 +110,21 @@ comp_renderer_set_equirect2_layer(struct comp_renderer *r, struct xrt_layer_data *data); #endif +/*! + * Allocate an internal array of per-layer data with the given number of elements. + * + * @public @memberof comp_renderer + * @ingroup comp_main + */ void comp_renderer_allocate_layers(struct comp_renderer *self, uint32_t num_layers); +/*! + * De-initialize and free internal array of per-layer data. + * + * @public @memberof comp_renderer + * @ingroup comp_main + */ void comp_renderer_destroy_layers(struct comp_renderer *self); diff --git a/src/xrt/compositor/main/comp_settings.c b/src/xrt/compositor/main/comp_settings.c index 33474dfef..f62208f5a 100644 --- a/src/xrt/compositor/main/comp_settings.c +++ b/src/xrt/compositor/main/comp_settings.c @@ -11,9 +11,10 @@ #include "comp_settings.h" // clang-format off -DEBUG_GET_ONCE_LOG_OPTION(log, "XRT_COMPOSITOR_LOG", U_LOGGING_WARN) +DEBUG_GET_ONCE_LOG_OPTION(log, "XRT_COMPOSITOR_LOG", U_LOGGING_INFO) DEBUG_GET_ONCE_BOOL_OPTION(print_modes, "XRT_COMPOSITOR_PRINT_MODES", false) DEBUG_GET_ONCE_BOOL_OPTION(force_randr, "XRT_COMPOSITOR_FORCE_RANDR", false) +DEBUG_GET_ONCE_BOOL_OPTION(force_wayland_direct, "XRT_COMPOSITOR_FORCE_WAYLAND_DIRECT", false) DEBUG_GET_ONCE_BOOL_OPTION(force_nvidia, "XRT_COMPOSITOR_FORCE_NVIDIA", false) DEBUG_GET_ONCE_OPTION(nvidia_display, "XRT_COMPOSITOR_FORCE_NVIDIA_DISPLAY", NULL) DEBUG_GET_ONCE_NUM_OPTION(vk_display, "XRT_COMPOSITOR_FORCE_VK_DISPLAY", -1) @@ -27,6 +28,7 @@ DEBUG_GET_ONCE_NUM_OPTION(scale_percentage, "XRT_COMPOSITOR_SCALE_PERCENTAGE", 1 DEBUG_GET_ONCE_BOOL_OPTION(xcb_fullscreen, "XRT_COMPOSITOR_XCB_FULLSCREEN", false) DEBUG_GET_ONCE_NUM_OPTION(xcb_display, "XRT_COMPOSITOR_XCB_DISPLAY", -1) DEBUG_GET_ONCE_NUM_OPTION(default_framerate, "XRT_COMPOSITOR_DEFAULT_FRAMERATE", 60) +DEBUG_GET_ONCE_BOOL_OPTION(compute, "XRT_COMPOSITOR_COMPUTE", false) // clang-format on void @@ -39,8 +41,15 @@ comp_settings_init(struct comp_settings *s, struct xrt_device *xdev) interval_ns = (1000 * 1000 * 1000) / default_framerate; } + s->use_compute = debug_get_bool_option_compute(); + + if (s->use_compute) { + s->color_format = VK_FORMAT_B8G8R8A8_UNORM; + } else { + s->color_format = VK_FORMAT_B8G8R8A8_SRGB; + } + s->display = debug_get_num_option_xcb_display(); - s->color_format = VK_FORMAT_B8G8R8A8_SRGB; s->color_space = VK_COLOR_SPACE_SRGB_NONLINEAR_KHR; s->present_mode = VK_PRESENT_MODE_FIFO_KHR; s->window_type = WINDOW_AUTO; @@ -70,6 +79,11 @@ comp_settings_init(struct comp_settings *s, struct xrt_device *xdev) s->window_type = WINDOW_DIRECT_RANDR; } + if (debug_get_bool_option_force_wayland_direct()) { + s->window_type = WINDOW_DIRECT_WAYLAND; + } + + if (debug_get_bool_option_force_xcb()) { s->window_type = WINDOW_XCB; // HMD screen tends to be much larger then monitors. diff --git a/src/xrt/compositor/main/comp_settings.h b/src/xrt/compositor/main/comp_settings.h index 52c6c451b..4186b437d 100644 --- a/src/xrt/compositor/main/comp_settings.h +++ b/src/xrt/compositor/main/comp_settings.h @@ -44,6 +44,7 @@ enum window_type WINDOW_AUTO, WINDOW_XCB, WINDOW_WAYLAND, + WINDOW_DIRECT_WAYLAND, WINDOW_DIRECT_RANDR, WINDOW_DIRECT_NVIDIA, WINDOW_ANDROID, @@ -61,6 +62,8 @@ struct comp_settings { int display; + bool use_compute; + VkFormat color_format; VkColorSpaceKHR color_space; VkPresentModeKHR present_mode; @@ -86,7 +89,7 @@ struct comp_settings bool wireframe; } debug; - //! Procentage to scale the viewport by. + //! Percentage to scale the viewport by. double viewport_scale; //! Not used with direct mode. diff --git a/src/xrt/compositor/main/comp_shaders.c b/src/xrt/compositor/main/comp_shaders.c index e0bdc2f85..ad8429d55 100644 --- a/src/xrt/compositor/main/comp_shaders.c +++ b/src/xrt/compositor/main/comp_shaders.c @@ -20,6 +20,9 @@ #pragma GCC diagnostic ignored "-Wnewline-eof" +#include "shaders/clear.comp.h" +#include "shaders/distortion.comp.h" +#include "shaders/distortion_timewarp.comp.h" #include "shaders/layer.frag.h" #include "shaders/layer.vert.h" #include "shaders/equirect1.frag.h" @@ -78,6 +81,21 @@ shader_load(struct vk_bundle *vk, const uint32_t *code, size_t size, VkShaderMod bool comp_shaders_load(struct vk_bundle *vk, struct comp_shaders *s) { + C(shader_load(vk, // vk_bundle + shaders_clear_comp, // data + sizeof(shaders_clear_comp), // size + &s->clear_comp)); // out + + C(shader_load(vk, // vk_bundle + shaders_distortion_comp, // data + sizeof(shaders_distortion_comp), // size + &s->distortion_comp)); // out + + C(shader_load(vk, // vk_bundle + shaders_distortion_timewarp_comp, // data + sizeof(shaders_distortion_timewarp_comp), // size + &s->distortion_timewarp_comp)); // out + C(shader_load(vk, // vk_bundle shaders_mesh_vert, // data sizeof(shaders_mesh_vert), // size @@ -128,6 +146,9 @@ comp_shaders_load(struct vk_bundle *vk, struct comp_shaders *s) void comp_shaders_close(struct vk_bundle *vk, struct comp_shaders *s) { + D(clear_comp); + D(distortion_comp); + D(distortion_timewarp_comp); D(mesh_vert); D(mesh_frag); D(equirect1_vert); diff --git a/src/xrt/compositor/main/comp_swapchain.c b/src/xrt/compositor/main/comp_swapchain.c index 71fd08b91..5d66109a2 100644 --- a/src/xrt/compositor/main/comp_swapchain.c +++ b/src/xrt/compositor/main/comp_swapchain.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -132,6 +132,7 @@ do_post_create_vulkan_setup(struct comp_compositor *c, const struct xrt_swapchain_create_info *info, struct comp_swapchain *sc) { + struct vk_bundle *vk = &c->vk; uint32_t num_images = sc->vkic.num_images; VkCommandBuffer cmd_buffer; @@ -172,9 +173,9 @@ do_post_create_vulkan_setup(struct comp_compositor *c, sc->images[i].views.no_alpha = U_TYPED_ARRAY_CALLOC(VkImageView, info->array_size); sc->images[i].array_size = info->array_size; - vk_create_sampler(&c->vk, VK_SAMPLER_ADDRESS_MODE_REPEAT, &sc->images[i].repeat_sampler); + vk_create_sampler(vk, VK_SAMPLER_ADDRESS_MODE_REPEAT, &sc->images[i].repeat_sampler); - vk_create_sampler(&c->vk, VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, &sc->images[i].sampler); + vk_create_sampler(vk, VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, &sc->images[i].sampler); for (uint32_t layer = 0; layer < info->array_size; ++layer) { @@ -186,9 +187,9 @@ do_post_create_vulkan_setup(struct comp_compositor *c, .layerCount = 1, }; - vk_create_view(&c->vk, sc->vkic.images[i].handle, (VkFormat)info->format, subresource_range, + vk_create_view(vk, sc->vkic.images[i].handle, (VkFormat)info->format, subresource_range, &sc->images[i].views.alpha[layer]); - vk_create_view_swizzle(&c->vk, sc->vkic.images[i].handle, format, subresource_range, components, + vk_create_view_swizzle(vk, sc->vkic.images[i].handle, format, subresource_range, components, &sc->images[i].views.no_alpha[layer]); } } @@ -205,7 +206,7 @@ do_post_create_vulkan_setup(struct comp_compositor *c, * */ - vk_init_cmd_buffer(&c->vk, &cmd_buffer); + vk_init_cmd_buffer(vk, &cmd_buffer); VkImageSubresourceRange subresource_range = { .aspectMask = aspect, @@ -216,12 +217,12 @@ do_post_create_vulkan_setup(struct comp_compositor *c, }; for (uint32_t i = 0; i < num_images; i++) { - vk_set_image_layout(&c->vk, cmd_buffer, sc->vkic.images[i].handle, 0, VK_ACCESS_SHADER_READ_BIT, + vk_set_image_layout(vk, cmd_buffer, sc->vkic.images[i].handle, 0, VK_ACCESS_SHADER_READ_BIT, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, subresource_range); } - vk_submit_cmd_buffer(&c->vk, cmd_buffer); + vk_submit_cmd_buffer(vk, cmd_buffer); } static void @@ -282,6 +283,7 @@ comp_swapchain_create(struct xrt_compositor *xc, struct xrt_swapchain **out_xsc) { struct comp_compositor *c = comp_compositor(xc); + struct vk_bundle *vk = &c->vk; uint32_t num_images = 3; VkResult ret; @@ -301,12 +303,12 @@ comp_swapchain_create(struct xrt_compositor *xc, struct comp_swapchain *sc = alloc_and_set_funcs(c, num_images); - COMP_DEBUG(c, "CREATE %p %dx%d %s", (void *)sc, // - info->width, info->height, // - vk_color_format_string(info->format)); + COMP_DEBUG(c, "CREATE %p %dx%d %s (%ld)", (void *)sc, // + info->width, info->height, // + vk_color_format_string(info->format), info->format); // Use the image helper to allocate the images. - ret = vk_ic_allocate(&c->vk, info, num_images, &sc->vkic); + ret = vk_ic_allocate(vk, info, num_images, &sc->vkic); if (ret == VK_ERROR_FEATURE_NOT_PRESENT) { free(sc); return XRT_ERROR_SWAPCHAIN_FLAG_VALID_BUT_UNSUPPORTED; @@ -321,15 +323,17 @@ comp_swapchain_create(struct xrt_compositor *xc, xrt_graphics_buffer_handle_t handles[ARRAY_SIZE(sc->vkic.images)]; - vk_ic_get_handles(&c->vk, &sc->vkic, ARRAY_SIZE(handles), handles); + vk_ic_get_handles(vk, &sc->vkic, ARRAY_SIZE(handles), handles); for (uint32_t i = 0; i < sc->vkic.num_images; i++) { sc->base.images[i].handle = handles[i]; sc->base.images[i].size = sc->vkic.images[i].size; + sc->base.images[i].use_dedicated_allocation = sc->vkic.images[i].use_dedicated_allocation; } do_post_create_vulkan_setup(c, info, sc); - *out_xsc = &sc->base.base; + // Correctly setup refcounts. + xrt_swapchain_reference(out_xsc, &sc->base.base); return XRT_SUCCESS; } @@ -342,6 +346,7 @@ comp_swapchain_import(struct xrt_compositor *xc, struct xrt_swapchain **out_xsc) { struct comp_compositor *c = comp_compositor(xc); + struct vk_bundle *vk = &c->vk; VkResult ret; struct comp_swapchain *sc = alloc_and_set_funcs(c, num_images); @@ -349,14 +354,15 @@ comp_swapchain_import(struct xrt_compositor *xc, COMP_DEBUG(c, "CREATE FROM NATIVE %p %dx%d", (void *)sc, info->width, info->height); // Use the image helper to get the images. - ret = vk_ic_from_natives(&c->vk, info, native_images, num_images, &sc->vkic); + ret = vk_ic_from_natives(vk, info, native_images, num_images, &sc->vkic); if (ret != VK_SUCCESS) { return XRT_ERROR_VULKAN; } do_post_create_vulkan_setup(c, info, sc); - *out_xsc = &sc->base.base; + // Correctly setup refcounts. + xrt_swapchain_reference(out_xsc, &sc->base.base); return XRT_SUCCESS; } diff --git a/src/xrt/compositor/main/comp_sync.c b/src/xrt/compositor/main/comp_sync.c new file mode 100644 index 000000000..2d852d5b4 --- /dev/null +++ b/src/xrt/compositor/main/comp_sync.c @@ -0,0 +1,124 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Sync code for the main compositor. + * @author Jakob Bornecrantz + * @ingroup comp_main + */ + +#include "util/u_misc.h" +#include "util/u_handles.h" + +#include "main/comp_compositor.h" + +#include +#include + +#include +#include + +#ifndef XRT_OS_WINDOWS +#include +#endif + + + +struct fence +{ + struct xrt_compositor_fence base; + struct comp_compositor *c; + + VkFence fence; +}; + + +/* + * + * Helper functions. + * + */ + + + +/* + * + * Fence member functions. + * + */ + +static xrt_result_t +fence_wait(struct xrt_compositor_fence *xcf, uint64_t timeout) +{ + COMP_TRACE_MARKER(); + + struct fence *f = (struct fence *)xcf; + struct vk_bundle *vk = &f->c->vk; + + // Count no handle as signled fence. + if (f->fence == VK_NULL_HANDLE) { + return XRT_SUCCESS; + } + + VkResult ret = vk->vkWaitForFences(vk->device, 1, &f->fence, VK_TRUE, timeout); + if (ret == VK_TIMEOUT) { + return XRT_SUCCESS; + } + if (ret != VK_SUCCESS) { + COMP_ERROR(f->c, "vkWaitForFences: %s", vk_result_string(ret)); + return XRT_ERROR_VULKAN; + } + + return XRT_SUCCESS; +} + +static void +fence_destroy(struct xrt_compositor_fence *xcf) +{ + COMP_TRACE_MARKER(); + + struct fence *f = (struct fence *)xcf; + struct vk_bundle *vk = &f->c->vk; + + if (f->fence != VK_NULL_HANDLE) { + vk->vkDestroyFence(vk->device, f->fence, NULL); + f->fence = VK_NULL_HANDLE; + } + + free(f); +} + + +/* + * + * Compositor function. + * + */ + +xrt_result_t +comp_compositor_import_fence(struct xrt_compositor *xc, + xrt_graphics_sync_handle_t handle, + struct xrt_compositor_fence **out_xcf) +{ + COMP_TRACE_MARKER(); + + struct comp_compositor *c = comp_compositor(xc); + struct vk_bundle *vk = &c->vk; + + VkFence fence = VK_NULL_HANDLE; + + VkResult ret = vk_create_fence_sync_from_native(vk, handle, &fence); + if (ret != VK_SUCCESS) { + return XRT_ERROR_VULKAN; + } + + struct fence *f = U_TYPED_CALLOC(struct fence); + f->base.wait = fence_wait; + f->base.destroy = fence_destroy; + f->fence = fence; + f->c = c; + + *out_xcf = &f->base; + + return XRT_SUCCESS; +} diff --git a/src/xrt/compositor/main/comp_target.h b/src/xrt/compositor/main/comp_target.h index 1c3b35f77..d72496a61 100644 --- a/src/xrt/compositor/main/comp_target.h +++ b/src/xrt/compositor/main/comp_target.h @@ -14,12 +14,33 @@ #include "vk/vk_helpers.h" +#include "util/u_trace_marker.h" + #ifdef __cplusplus extern "C" { #endif +/*! + * For marking timepoints on a frame's lifetime, not a async event. + */ +enum comp_target_timing_point +{ + COMP_TARGET_TIMING_POINT_WAKE_UP, //init_pre_vulkan(ct); } @@ -130,9 +233,25 @@ comp_target_init_pre_vulkan(struct comp_target *ct) static inline bool comp_target_init_post_vulkan(struct comp_target *ct, uint32_t preferred_width, uint32_t preferred_height) { + COMP_TRACE_MARKER(); + return ct->init_post_vulkan(ct, preferred_width, preferred_height); } +/*! + * @copydoc comp_target::check_ready + * + * @public @memberof comp_target + * @ingroup comp_main + */ +static inline bool +comp_target_check_ready(struct comp_target *ct) +{ + COMP_TRACE_MARKER(); + + return ct->check_ready(ct); +} + /*! * @copydoc comp_target::create_images * @@ -145,10 +264,33 @@ comp_target_create_images(struct comp_target *ct, uint32_t preferred_height, VkFormat preferred_color_format, VkColorSpaceKHR preferred_color_space, + VkImageUsageFlags image_usage, VkPresentModeKHR present_mode) { - ct->create_images(ct, preferred_width, preferred_height, preferred_color_format, preferred_color_space, - present_mode); + COMP_TRACE_MARKER(); + + ct->create_images( // + ct, // + preferred_width, // + preferred_height, // + preferred_color_format, // + preferred_color_space, // + image_usage, // + present_mode); // +} + +/*! + * @copydoc comp_target::has_images + * + * @public @memberof comp_target + * @ingroup comp_main + */ +static inline bool +comp_target_has_images(struct comp_target *ct) +{ + COMP_TRACE_MARKER(); + + return ct->has_images(ct); } /*! @@ -160,6 +302,8 @@ comp_target_create_images(struct comp_target *ct, static inline VkResult comp_target_acquire(struct comp_target *ct, VkSemaphore semaphore, uint32_t *out_index) { + COMP_TRACE_MARKER(); + return ct->acquire(ct, semaphore, out_index); } @@ -170,10 +314,23 @@ comp_target_acquire(struct comp_target *ct, VkSemaphore semaphore, uint32_t *out * @ingroup comp_main */ static inline VkResult -comp_target_present(struct comp_target *ct, VkQueue queue, uint32_t index, VkSemaphore semaphore) +comp_target_present(struct comp_target *ct, + VkQueue queue, + uint32_t index, + VkSemaphore semaphore, + uint64_t desired_present_time_ns, + uint64_t present_slop_ns) { - return ct->present(ct, queue, index, semaphore); + COMP_TRACE_MARKER(); + + return ct->present( // + ct, // + queue, // + index, // + semaphore, // + desired_present_time_ns, // + present_slop_ns); // } /*! @@ -185,9 +342,95 @@ comp_target_present(struct comp_target *ct, VkQueue queue, uint32_t index, VkSem static inline void comp_target_flush(struct comp_target *ct) { + COMP_TRACE_MARKER(); + ct->flush(ct); } +/*! + * @copydoc comp_target::calc_frame_timings + * + * @public @memberof comp_target + * @ingroup comp_main + */ +static inline void +comp_target_calc_frame_timings(struct comp_target *ct, + int64_t *out_frame_id, + uint64_t *out_wake_up_time_ns, + uint64_t *out_desired_present_time_ns, + uint64_t *out_present_slop_ns, + uint64_t *out_predicted_display_time_ns) +{ + COMP_TRACE_MARKER(); + + ct->calc_frame_timings( // + ct, // + out_frame_id, // + out_wake_up_time_ns, // + out_desired_present_time_ns, // + out_present_slop_ns, // + out_predicted_display_time_ns); // +} + +/*! + * Quick helper for marking wake up. + * @copydoc comp_target::mark_timing_point + * + * @public @memberof comp_target + * @ingroup comp_main + */ +static inline void +comp_target_mark_wake_up(struct comp_target *ct, int64_t frame_id, uint64_t when_woke_ns) +{ + COMP_TRACE_MARKER(); + + ct->mark_timing_point(ct, COMP_TARGET_TIMING_POINT_WAKE_UP, frame_id, when_woke_ns); +} + +/*! + * Quick helper for marking begin. + * @copydoc comp_target::mark_timing_point + * + * @public @memberof comp_target + * @ingroup comp_main + */ +static inline void +comp_target_mark_begin(struct comp_target *ct, int64_t frame_id, uint64_t when_began_ns) +{ + COMP_TRACE_MARKER(); + + ct->mark_timing_point(ct, COMP_TARGET_TIMING_POINT_BEGIN, frame_id, when_began_ns); +} + +/*! + * Quick helper for marking submit. + * @copydoc comp_target::mark_timing_point + * + * @public @memberof comp_target + * @ingroup comp_main + */ +static inline void +comp_target_mark_submit(struct comp_target *ct, int64_t frame_id, uint64_t when_submitted_ns) +{ + COMP_TRACE_MARKER(); + + ct->mark_timing_point(ct, COMP_TARGET_TIMING_POINT_SUBMIT, frame_id, when_submitted_ns); +} + +/*! + * @copydoc comp_target::update_timings + * + * @public @memberof comp_target + * @ingroup comp_main + */ +static inline VkResult +comp_target_update_timings(struct comp_target *ct) +{ + COMP_TRACE_MARKER(); + + return ct->update_timings(ct); +} + /*! * @copydoc comp_target::set_title * @@ -197,6 +440,8 @@ comp_target_flush(struct comp_target *ct) static inline void comp_target_set_title(struct comp_target *ct, const char *title) { + COMP_TRACE_MARKER(); + ct->set_title(ct, title); } diff --git a/src/xrt/compositor/main/comp_target_swapchain.c b/src/xrt/compositor/main/comp_target_swapchain.c index cd016fd4b..94e7c28cc 100644 --- a/src/xrt/compositor/main/comp_target_swapchain.c +++ b/src/xrt/compositor/main/comp_target_swapchain.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -8,7 +8,10 @@ * @ingroup comp_main */ +#include "xrt/xrt_config_os.h" + #include "util/u_misc.h" +#include "util/u_timing.h" #include "main/comp_compositor.h" #include "main/comp_target_swapchain.h" @@ -17,6 +20,7 @@ #include #include #include +#include /* @@ -31,10 +35,11 @@ * interested in should support one these. */ static VkFormat preferred_color_formats[] = { - VK_FORMAT_R8G8B8A8_SRGB, - VK_FORMAT_R8G8B8A8_UNORM, - VK_FORMAT_B8G8R8A8_UNORM, - VK_FORMAT_A8B8G8R8_UNORM_PACK32, + VK_FORMAT_B8G8R8A8_SRGB, // + VK_FORMAT_R8G8B8A8_SRGB, // + VK_FORMAT_B8G8R8A8_UNORM, // + VK_FORMAT_R8G8B8A8_UNORM, // + VK_FORMAT_A8B8G8R8_UNORM_PACK32, // Just in case. }; @@ -56,8 +61,8 @@ comp_target_swapchain_destroy_old(struct comp_target_swapchain *cts, VkSwapchain static VkExtent2D comp_target_swapchain_select_extent(struct comp_target_swapchain *cts, VkSurfaceCapabilitiesKHR caps, - uint32_t width, - uint32_t height); + uint32_t preferred_width, + uint32_t preferred_height); static bool _find_surface_format(struct comp_target_swapchain *cts, VkSurfaceKHR surface, VkSurfaceFormatKHR *format); @@ -68,7 +73,7 @@ _check_surface_present_mode(struct comp_target_swapchain *cts, VkSurfaceKHR surf /* * - * Functions! + * Vulkan functions. * */ @@ -78,12 +83,13 @@ get_vk(struct comp_target_swapchain *cts) return &cts->base.c->vk; } -void +static void comp_target_swapchain_create_images(struct comp_target *ct, - uint32_t width, - uint32_t height, + uint32_t preferred_width, + uint32_t preferred_height, VkFormat color_format, VkColorSpaceKHR color_space, + VkImageUsageFlags image_usage, VkPresentModeKHR present_mode) { struct comp_target_swapchain *cts = (struct comp_target_swapchain *)ct; @@ -91,6 +97,14 @@ comp_target_swapchain_create_images(struct comp_target *ct, VkBool32 supported; VkResult ret; + // Some platforms really don't like the display_timing code. + bool use_display_timing_if_available = cts->timing_usage == COMP_TARGET_USE_DISPLAY_IF_AVAILABLE; + if (cts->uft == NULL && use_display_timing_if_available && vk->has_GOOGLE_display_timing) { + u_ft_display_timing_create(ct->c->settings.nominal_frame_interval_ns, &cts->uft); + } else if (cts->uft == NULL) { + u_ft_fake_create(ct->c->settings.nominal_frame_interval_ns, &cts->uft); + } + // Free old image views. comp_target_swapchain_destroy_image_views(cts); @@ -104,12 +118,15 @@ comp_target_swapchain_create_images(struct comp_target *ct, // Sanity check. - ret = vk->vkGetPhysicalDeviceSurfaceSupportKHR(vk->physical_device, 0, cts->surface.handle, &supported); - if (!supported) { - COMP_ERROR(ct->c, - "vkGetPhysicalDeviceSurfaceSupportKHR: surface not " - "supported! '%s'", - vk_result_string(ret)); + ret = vk->vkGetPhysicalDeviceSurfaceSupportKHR( // + vk->physical_device, // physicalDevice + vk->queue_family_index, // queueFamilyIndex + cts->surface.handle, // surface + &supported); // pSupported + if (ret != VK_SUCCESS) { + COMP_ERROR(ct->c, "vkGetPhysicalDeviceSurfaceSupportKHR: %s", vk_result_string(ret)); + } else if (!supported) { + COMP_ERROR(ct->c, "vkGetPhysicalDeviceSurfaceSupportKHR: Surface not supported!"); } // More sanity checks. @@ -138,13 +155,11 @@ comp_target_swapchain_create_images(struct comp_target *ct, } // Get the extents of the swapchain. - VkExtent2D extent = comp_target_swapchain_select_extent(cts, surface_caps, width, height); + VkExtent2D extent = comp_target_swapchain_select_extent(cts, surface_caps, preferred_width, preferred_height); if (surface_caps.currentTransform & VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR || surface_caps.currentTransform & VK_SURFACE_TRANSFORM_ROTATE_270_BIT_KHR) { - COMP_DEBUG(ct->c, - "Swapping width and height," - "since we are going to pre rotate"); + COMP_DEBUG(ct->c, "Swapping width and height, since we are going to pre rotate"); uint32_t w2 = extent.width; uint32_t h2 = extent.height; extent.width = h2; @@ -164,7 +179,7 @@ comp_target_swapchain_create_images(struct comp_target *ct, .height = extent.height, }, .imageArrayLayers = 1, - .imageUsage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, + .imageUsage = image_usage, .imageSharingMode = VK_SHARING_MODE_EXCLUSIVE, .queueFamilyIndexCount = 0, .preTransform = surface_caps.currentTransform, @@ -200,24 +215,28 @@ comp_target_swapchain_create_images(struct comp_target *ct, static VkExtent2D comp_target_swapchain_select_extent(struct comp_target_swapchain *cts, VkSurfaceCapabilitiesKHR caps, - uint32_t width, - uint32_t height) + uint32_t preferred_width, + uint32_t preferred_height) { // If width (and height) equals the special value 0xFFFFFFFF, // the size of the surface will be set by the swapchain if (caps.currentExtent.width == (uint32_t)-1) { + assert(preferred_width > 0 && preferred_height > 0); + VkExtent2D extent = { - .width = width, - .height = height, + .width = preferred_width, + .height = preferred_height, }; return extent; } - if (caps.currentExtent.width != width || caps.currentExtent.height != height) { - COMP_DEBUG(cts->base.c, - "Using swap chain extent dimensions %dx%d instead " - "of requested %dx%d.", - caps.currentExtent.width, caps.currentExtent.height, width, height); + if (caps.currentExtent.width != preferred_width || // + caps.currentExtent.height != preferred_height) { + COMP_DEBUG(cts->base.c, "Using swap chain extent dimensions %dx%d instead of requested %dx%d.", + caps.currentExtent.width, // + caps.currentExtent.height, // + preferred_width, // + preferred_height); // } return caps.currentExtent; @@ -233,30 +252,61 @@ comp_target_swapchain_destroy_old(struct comp_target_swapchain *cts, VkSwapchain } } -VkResult +static bool +comp_target_swapchain_has_images(struct comp_target *ct) +{ + struct comp_target_swapchain *cts = (struct comp_target_swapchain *)ct; + return cts->surface.handle != VK_NULL_HANDLE && cts->swapchain.handle != VK_NULL_HANDLE; +} + +static VkResult comp_target_swapchain_acquire_next_image(struct comp_target *ct, VkSemaphore semaphore, uint32_t *out_index) { struct comp_target_swapchain *cts = (struct comp_target_swapchain *)ct; struct vk_bundle *vk = get_vk(cts); + if (!comp_target_swapchain_has_images(ct)) { + //! @todo what error to return here? + return VK_ERROR_INITIALIZATION_FAILED; + } return vk->vkAcquireNextImageKHR( // vk->device, // device - cts->swapchain.handle, // timeout + cts->swapchain.handle, // swapchain UINT64_MAX, // timeout semaphore, // semaphore VK_NULL_HANDLE, // fence out_index); // pImageIndex } -VkResult -comp_target_swapchain_present(struct comp_target *ct, VkQueue queue, uint32_t index, VkSemaphore semaphore) +static VkResult +comp_target_swapchain_present(struct comp_target *ct, + VkQueue queue, + uint32_t index, + VkSemaphore semaphore, + uint64_t desired_present_time_ns, + uint64_t present_slop_ns) { struct comp_target_swapchain *cts = (struct comp_target_swapchain *)ct; struct vk_bundle *vk = get_vk(cts); + assert(cts->current_frame_id >= 0); + assert(cts->current_frame_id <= UINT32_MAX); + + VkPresentTimeGOOGLE times = { + .presentID = (uint32_t)cts->current_frame_id, + .desiredPresentTime = desired_present_time_ns - present_slop_ns, + }; + + VkPresentTimesInfoGOOGLE timings = { + .sType = VK_STRUCTURE_TYPE_PRESENT_TIMES_INFO_GOOGLE, + .swapchainCount = 1, + .pTimes = ×, + }; + VkPresentInfoKHR presentInfo = { .sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR, + .pNext = vk->has_GOOGLE_display_timing ? &timings : NULL, .waitSemaphoreCount = 1, .pWaitSemaphores = &semaphore, .swapchainCount = 1, @@ -267,6 +317,13 @@ comp_target_swapchain_present(struct comp_target *ct, VkQueue queue, uint32_t in return vk->vkQueuePresentKHR(queue, &presentInfo); } +static bool +comp_target_swapchain_check_ready(struct comp_target *ct) +{ + struct comp_target_swapchain *cts = (struct comp_target_swapchain *)ct; + return cts->surface.handle != VK_NULL_HANDLE; +} + static bool _find_surface_format(struct comp_target_swapchain *cts, VkSurfaceKHR surface, VkSurfaceFormatKHR *format) { @@ -285,6 +342,12 @@ _find_surface_format(struct comp_target_swapchain *cts, VkSurfaceKHR surface, Vk return false; } + // Dump formats + for (uint32_t i = 0; i < num_formats; i++) { + COMP_SPEW(cts->base.c, "VkSurfaceFormatKHR: %i [%s, %s]", i, vk_color_format_string(formats[i].format), + vk_color_space_string(formats[i].colorSpace)); + } + VkSurfaceFormatKHR *formats_for_colorspace = NULL; formats_for_colorspace = U_TYPED_ARRAY_CALLOC(VkSurfaceFormatKHR, num_formats); @@ -294,6 +357,7 @@ _find_surface_format(struct comp_target_swapchain *cts, VkSurfaceKHR surface, Vk // Gather formats that match our color space, we will select // from these in preference to others. + for (uint32_t i = 0; i < num_formats; i++) { if (formats[i].colorSpace == cts->preferred.color_space) { formats_for_colorspace[num_formats_cs] = formats[i]; @@ -342,8 +406,7 @@ _find_surface_format(struct comp_target_swapchain *cts, VkSurfaceKHR surface, Vk if (formats[i].format == preferred_color_formats[j]) { *format = formats_for_colorspace[i]; COMP_ERROR(cts->base.c, - "Returning known-wrong color " - "space! Color shift may occur."); + "Returning known-wrong color space! Color shift may occur."); goto cleanup; } } @@ -353,8 +416,7 @@ _find_surface_format(struct comp_target_swapchain *cts, VkSurfaceKHR surface, Vk // list of preferred formats, but its something. *format = formats[0]; COMP_ERROR(cts->base.c, - "Returning fallback format! cue up some Kenny " - "Loggins, cos we're in the DANGER ZONE!"); + "Returning fallback format! cue up some Kenny Loggins, cos we're in the DANGER ZONE!"); goto cleanup; } @@ -364,6 +426,16 @@ _find_surface_format(struct comp_target_swapchain *cts, VkSurfaceKHR surface, Vk cleanup: free(formats_for_colorspace); free(formats); + + COMP_DEBUG(cts->base.c, + "VkSurfaceFormatKHR" + "\n\tpicked: [format = %s, colorSpace = %s]" + "\n\tpreferred: [format = %s, colorSpace = %s]", + vk_color_format_string(format->format), // + vk_color_space_string(format->colorSpace), // + vk_color_format_string(cts->preferred.color_format), // + vk_color_space_string(cts->preferred.color_space)); // + return true; error: @@ -459,13 +531,133 @@ comp_target_swapchain_create_image_views(struct comp_target_swapchain *cts) for (uint32_t i = 0; i < cts->base.num_images; i++) { cts->base.images[i].handle = images[i]; - vk_create_view(vk, cts->base.images[i].handle, cts->surface.format.format, subresource_range, - &cts->base.images[i].view); + vk_create_view( // + vk, // vk_bundle + cts->base.images[i].handle, // image + cts->surface.format.format, // format + subresource_range, // subresource_range + &cts->base.images[i].view); // out_view } free(images); } + +/* + * + * Timing functions. + * + */ + +static void +comp_target_swapchain_calc_frame_timings(struct comp_target *ct, + int64_t *out_frame_id, + uint64_t *out_wake_up_time_ns, + uint64_t *out_desired_present_time_ns, + uint64_t *out_present_slop_ns, + uint64_t *out_predicted_display_time_ns) +{ + struct comp_target_swapchain *cts = (struct comp_target_swapchain *)ct; + + int64_t frame_id = -1; + uint64_t wake_up_time_ns = 0; + uint64_t desired_present_time_ns = 0; + uint64_t present_slop_ns = 0; + uint64_t predicted_display_time_ns = 0; + uint64_t predicted_display_period_ns = 0; + uint64_t min_display_period_ns = 0; + + u_ft_predict(cts->uft, // + &frame_id, // + &wake_up_time_ns, // + &desired_present_time_ns, // + &present_slop_ns, // + &predicted_display_time_ns, // + &predicted_display_period_ns, // + &min_display_period_ns); // + + cts->current_frame_id = frame_id; + + *out_frame_id = frame_id; + *out_wake_up_time_ns = wake_up_time_ns; + *out_desired_present_time_ns = desired_present_time_ns; + *out_predicted_display_time_ns = predicted_display_time_ns; + *out_present_slop_ns = present_slop_ns; +} + +static void +comp_target_swapchain_mark_timing_point(struct comp_target *ct, + enum comp_target_timing_point point, + int64_t frame_id, + uint64_t when_ns) +{ + struct comp_target_swapchain *cts = (struct comp_target_swapchain *)ct; + assert(frame_id == cts->current_frame_id); + + switch (point) { + case COMP_TARGET_TIMING_POINT_WAKE_UP: + u_ft_mark_point(cts->uft, U_TIMING_POINT_WAKE_UP, cts->current_frame_id, when_ns); + break; + case COMP_TARGET_TIMING_POINT_BEGIN: + u_ft_mark_point(cts->uft, U_TIMING_POINT_BEGIN, cts->current_frame_id, when_ns); + break; + case COMP_TARGET_TIMING_POINT_SUBMIT: + u_ft_mark_point(cts->uft, U_TIMING_POINT_SUBMIT, cts->current_frame_id, when_ns); + break; + default: assert(false); + } +} + +static VkResult +comp_target_swapchain_update_timings(struct comp_target *ct) +{ + struct comp_target_swapchain *cts = (struct comp_target_swapchain *)ct; + struct vk_bundle *vk = get_vk(cts); + + if (!vk->has_GOOGLE_display_timing) { + return VK_SUCCESS; + } + + if (cts->swapchain.handle == VK_NULL_HANDLE) { + return VK_SUCCESS; + } + + uint32_t count = 0; + vk->vkGetPastPresentationTimingGOOGLE( // + vk->device, // + cts->swapchain.handle, // + &count, // + NULL); // + if (count <= 0) { + return VK_SUCCESS; + } + + VkPastPresentationTimingGOOGLE *timings = U_TYPED_ARRAY_CALLOC(VkPastPresentationTimingGOOGLE, count); + vk->vkGetPastPresentationTimingGOOGLE( // + vk->device, // + cts->swapchain.handle, // + &count, // + timings); // + + for (uint32_t i = 0; i < count; i++) { + u_ft_info(cts->uft, // + timings[i].presentID, // + timings[i].desiredPresentTime, // + timings[i].actualPresentTime, // + timings[i].earliestPresentTime, // + timings[i].presentMargin); // + } + free(timings); + return VK_SUCCESS; +} + + +/* + * + * 'Exported' functions. + * + */ + void comp_target_swapchain_cleanup(struct comp_target_swapchain *cts) { @@ -488,12 +680,21 @@ comp_target_swapchain_cleanup(struct comp_target_swapchain *cts) NULL); // cts->swapchain.handle = VK_NULL_HANDLE; } + + u_ft_destroy(&cts->uft); } void -comp_target_swapchain_init_set_fnptrs(struct comp_target_swapchain *cts) +comp_target_swapchain_init_and_set_fnptrs(struct comp_target_swapchain *cts, + enum comp_target_display_timing_usage timing_usage) { + cts->timing_usage = timing_usage; + cts->base.check_ready = comp_target_swapchain_check_ready; cts->base.create_images = comp_target_swapchain_create_images; + cts->base.has_images = comp_target_swapchain_has_images; cts->base.acquire = comp_target_swapchain_acquire_next_image; cts->base.present = comp_target_swapchain_present; + cts->base.calc_frame_timings = comp_target_swapchain_calc_frame_timings; + cts->base.mark_timing_point = comp_target_swapchain_mark_timing_point; + cts->base.update_timings = comp_target_swapchain_update_timings; } diff --git a/src/xrt/compositor/main/comp_target_swapchain.h b/src/xrt/compositor/main/comp_target_swapchain.h index 6624e6677..c1e146ab1 100644 --- a/src/xrt/compositor/main/comp_target_swapchain.h +++ b/src/xrt/compositor/main/comp_target_swapchain.h @@ -26,6 +26,8 @@ extern "C" { * */ +struct u_frame_timing; + /*! * Wraps and manage VkSwapchainKHR and VkSurfaceKHR, used by @ref comp code. * @@ -36,6 +38,15 @@ struct comp_target_swapchain //! Base target. struct comp_target base; + //! Frame timing tracker. + struct u_frame_timing *uft; + + //! If we should use display timing. + enum comp_target_display_timing_usage timing_usage; + + //! Also works as a frame index. + int64_t current_frame_id; + struct { VkSwapchainKHR handle; @@ -65,30 +76,38 @@ struct comp_target_swapchain */ /*! - * Pre Vulkan initialisation, sets function pointers. + * @brief Pre Vulkan initialisation, sets function pointers. + * + * Call from the creation function for your "subclass", after allocating. + * + * Initializes these function pointers, all other methods of @ref comp_target are the responsibility of the caller (the + * "subclass"): + * + * - comp_target::check_ready + * - comp_target::create_images + * - comp_target::has_images + * - comp_target::acquire + * - comp_target::present + * - comp_target::calc_frame_timings + * - comp_target::mark_timing_point + * - comp_target::update_timings + * + * Also sets comp_target_swapchain::timing_usage to the provided value. + * + * @protected @memberof comp_target_swapchain * * @ingroup comp_main */ void -comp_target_swapchain_init_set_fnptrs(struct comp_target_swapchain *cts); - -/*! - * See comp_target::create_images. - * - * @ingroup comp_main - */ -void -comp_target_swapchain_create_images(struct comp_target *ct, - uint32_t width, - uint32_t height, - VkFormat color_format, - VkColorSpaceKHR color_space, - VkPresentModeKHR present_mode); +comp_target_swapchain_init_and_set_fnptrs(struct comp_target_swapchain *cts, + enum comp_target_display_timing_usage timing_usage); /*! * Free all managed resources on the given @ref comp_target_swapchain, * does not free the struct itself. * + * @protected @memberof comp_target_swapchain + * * @ingroup comp_main */ void diff --git a/src/xrt/compositor/main/comp_window.h b/src/xrt/compositor/main/comp_window.h index bb799e80e..60d3178db 100644 --- a/src/xrt/compositor/main/comp_window.h +++ b/src/xrt/compositor/main/comp_window.h @@ -46,6 +46,15 @@ comp_window_xcb_create(struct comp_compositor *c); */ struct comp_target * comp_window_wayland_create(struct comp_compositor *c); + +/*! + * Create a direct surface to a HMD via Wayland. + * + * @ingroup comp + */ +struct comp_target * +comp_window_direct_wayland_create(struct comp_compositor *c); + #endif #ifdef VK_USE_PLATFORM_XLIB_XRANDR_EXT diff --git a/src/xrt/compositor/main/comp_window_android.c b/src/xrt/compositor/main/comp_window_android.c index 187a01a54..bc8967552 100644 --- a/src/xrt/compositor/main/comp_window_android.c +++ b/src/xrt/compositor/main/comp_window_android.c @@ -167,7 +167,8 @@ comp_window_android_create(struct comp_compositor *c) { struct comp_window_android *w = U_TYPED_CALLOC(struct comp_window_android); - comp_target_swapchain_init_set_fnptrs(&w->base); + // The display timing code hasn't been tested on Android and may be broken. + comp_target_swapchain_init_and_set_fnptrs(&w->base, COMP_TARGET_FORCE_FAKE_DISPLAY_TIMING); w->base.base.name = "Android"; w->base.base.destroy = comp_window_android_destroy; diff --git a/src/xrt/compositor/main/comp_window_direct.c b/src/xrt/compositor/main/comp_window_direct.c index 8b5cec0c8..43a0aad3c 100644 --- a/src/xrt/compositor/main/comp_window_direct.c +++ b/src/xrt/compositor/main/comp_window_direct.c @@ -49,7 +49,7 @@ choose_best_vk_mode_auto(struct comp_target *ct, VkDisplayModePropertiesKHR *mod } VkDisplayModeParametersKHR best = mode_properties[best_index].parameters; COMP_DEBUG(ct->c, "Auto choosing Vk direct mode %d: %dx%d@%.2f", best_index, best.visibleRegion.width, - best.visibleRegion.width, (float)best.refreshRate / 1000.); + best.visibleRegion.width, (float)best.refreshRate / 1000.f); return best_index; } @@ -61,7 +61,7 @@ print_modes(struct comp_target *ct, VkDisplayModePropertiesKHR *mode_properties, VkDisplayModePropertiesKHR props = mode_properties[i]; uint16_t width = props.parameters.visibleRegion.width; uint16_t height = props.parameters.visibleRegion.height; - float refresh = (float)props.parameters.refreshRate / 1000.; + float refresh = (float)props.parameters.refreshRate / 1000.f; COMP_PRINT_MODE(ct->c, "| %2d | %dx%d@%.2f", i, width, height, refresh); } @@ -205,6 +205,8 @@ comp_window_direct_create_surface(struct comp_target_swapchain *cts, return result; } +#ifdef VK_USE_PLATFORM_XLIB_XRANDR_EXT + int comp_window_direct_connect(struct comp_target_swapchain *cts, Display **dpy) { @@ -258,3 +260,5 @@ comp_window_direct_init_swapchain( return true; } + +#endif diff --git a/src/xrt/compositor/main/comp_window_direct.h b/src/xrt/compositor/main/comp_window_direct.h index 2608e0229..d1c37bfe0 100644 --- a/src/xrt/compositor/main/comp_window_direct.h +++ b/src/xrt/compositor/main/comp_window_direct.h @@ -25,6 +25,8 @@ comp_window_direct_create_surface(struct comp_target_swapchain *cts, uint32_t width, uint32_t height); +#ifdef VK_USE_PLATFORM_XLIB_XRANDR_EXT + int comp_window_direct_connect(struct comp_target_swapchain *cts, Display **dpy); @@ -35,6 +37,8 @@ bool comp_window_direct_init_swapchain( struct comp_target_swapchain *cts, Display *dpy, VkDisplayKHR display, uint32_t width, uint32_t height); +#endif + #ifdef __cplusplus } #endif diff --git a/src/xrt/compositor/main/comp_window_direct_nvidia.c b/src/xrt/compositor/main/comp_window_direct_nvidia.c index fa1f0aeb0..b142c65f6 100644 --- a/src/xrt/compositor/main/comp_window_direct_nvidia.c +++ b/src/xrt/compositor/main/comp_window_direct_nvidia.c @@ -86,7 +86,8 @@ comp_window_direct_nvidia_create(struct comp_compositor *c) { struct comp_window_direct_nvidia *w = U_TYPED_CALLOC(struct comp_window_direct_nvidia); - comp_target_swapchain_init_set_fnptrs(&w->base); + // The display timing code hasn't been tested on nVidia and may be broken. + comp_target_swapchain_init_and_set_fnptrs(&w->base, COMP_TARGET_FORCE_FAKE_DISPLAY_TIMING); w->base.base.name = "direct"; w->base.base.destroy = comp_window_direct_nvidia_destroy; @@ -162,9 +163,10 @@ static bool comp_window_direct_nvidia_init(struct comp_target *ct) { struct comp_window_direct_nvidia *w_direct = (struct comp_window_direct_nvidia *)ct; + struct vk_bundle *vk = &ct->c->vk; // Sanity check. - if (ct->c->vk.instance == VK_NULL_HANDLE) { + if (vk->instance == VK_NULL_HANDLE) { COMP_ERROR(ct->c, "Vulkan not initialized before NVIDIA init!"); return false; } @@ -174,13 +176,10 @@ comp_window_direct_nvidia_init(struct comp_target *ct) return false; } - struct vk_bundle comp_vk = ct->c->vk; - // find our display using nvidia whitelist, enumerate its modes, and // pick the best one get a list of attached displays uint32_t display_count; - if (comp_vk.vkGetPhysicalDeviceDisplayPropertiesKHR(comp_vk.physical_device, &display_count, NULL) != - VK_SUCCESS) { + if (vk->vkGetPhysicalDeviceDisplayPropertiesKHR(vk->physical_device, &display_count, NULL) != VK_SUCCESS) { COMP_ERROR(ct->c, "Failed to get vulkan display count"); return false; } @@ -192,8 +191,8 @@ comp_window_direct_nvidia_init(struct comp_target *ct) struct VkDisplayPropertiesKHR *display_props = U_TYPED_ARRAY_CALLOC(VkDisplayPropertiesKHR, display_count); - if (display_props && comp_vk.vkGetPhysicalDeviceDisplayPropertiesKHR(comp_vk.physical_device, &display_count, - display_props) != VK_SUCCESS) { + if (display_props && vk->vkGetPhysicalDeviceDisplayPropertiesKHR(vk->physical_device, &display_count, + display_props) != VK_SUCCESS) { COMP_ERROR(ct->c, "Failed to get display properties"); free(display_props); return false; diff --git a/src/xrt/compositor/main/comp_window_direct_randr.c b/src/xrt/compositor/main/comp_window_direct_randr.c index ed5dd7287..d42e63438 100644 --- a/src/xrt/compositor/main/comp_window_direct_randr.c +++ b/src/xrt/compositor/main/comp_window_direct_randr.c @@ -113,7 +113,8 @@ comp_window_direct_randr_create(struct comp_compositor *c) { struct comp_window_direct_randr *w = U_TYPED_CALLOC(struct comp_window_direct_randr); - comp_target_swapchain_init_set_fnptrs(&w->base); + // Display timing is tested and know working. + comp_target_swapchain_init_and_set_fnptrs(&w->base, COMP_TARGET_USE_DISPLAY_IF_AVAILABLE); w->base.base.name = "direct"; w->base.base.destroy = comp_window_direct_randr_destroy; diff --git a/src/xrt/compositor/main/comp_window_direct_wayland.c b/src/xrt/compositor/main/comp_window_direct_wayland.c new file mode 100644 index 000000000..323074f2c --- /dev/null +++ b/src/xrt/compositor/main/comp_window_direct_wayland.c @@ -0,0 +1,461 @@ +// Copyright 2019, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Wayland direct mode code. + * @author Drew DeVault + * @author Lubosz Sarnecki + * @author Simon Zeni + * @ingroup comp + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "drm-lease-v1-client-protocol.h" +#include "xrt/xrt_compiler.h" +#include "main/comp_window.h" +#include "main/comp_window_direct.h" + +#ifndef VK_EXT_acquire_drm_display +#error "Wayland direct requires the Vulkan extension VK_EXT_acquire_drm_display" +#endif + +struct direct_wayland_lease +{ + struct comp_window_direct_wayland *w; + + int leased_fd; + bool finished; + struct wp_drm_lease_v1 *lease; +}; + +struct direct_wayland_lease_connector +{ + struct comp_window_direct_wayland *w; + + uint32_t id; + char *name, *description; + struct wp_drm_lease_connector_v1 *connector; + struct direct_wayland_lease_device *dev; + struct direct_wayland_lease_connector *next; +}; + +struct direct_wayland_lease_device +{ + struct comp_window_direct_wayland *w; + + int drm_fd; + char *path; + bool done; + struct wp_drm_lease_device_v1 *device; + struct direct_wayland_lease_connector *connectors; + struct direct_wayland_lease_device *next; +}; + +struct comp_window_direct_wayland +{ + struct comp_target_swapchain base; + + struct wl_display *display; + struct direct_wayland_lease_device *devices; + struct direct_wayland_lease *lease; +}; + +static void +direct_wayland_lease_device_destroy(struct direct_wayland_lease_device *dev) +{ + struct direct_wayland_lease_connector *conn = dev->connectors, *next_conn; + while (conn) { + next_conn = conn->next; + + wp_drm_lease_connector_v1_destroy(conn->connector); + + free(conn->name); + free(conn->description); + free(conn); + + conn = next_conn; + } + + wp_drm_lease_device_v1_destroy(dev->device); + close(dev->drm_fd); + + free(dev->path); + free(dev); +} + +static void +comp_window_direct_wayland_destroy(struct comp_target *w) +{ + struct comp_window_direct_wayland *w_wayland = (struct comp_window_direct_wayland *)w; + + comp_target_swapchain_cleanup(&w_wayland->base); + + struct direct_wayland_lease_device *dev = w_wayland->devices, *next_dev; + while (dev) { + next_dev = dev->next; + direct_wayland_lease_device_destroy(dev); + dev = next_dev; + } + + if (w_wayland->lease) { + close(w_wayland->lease->leased_fd); + wp_drm_lease_v1_destroy(w_wayland->lease->lease); + free(w_wayland->lease); + } + + if (w_wayland->display) { + wl_display_disconnect(w_wayland->display); + } + free(w_wayland); +} + +static inline struct vk_bundle * +get_vk(struct comp_window_direct_wayland *cww) +{ + return &cww->base.base.c->vk; +} + +static void +_lease_fd(void *data, struct wp_drm_lease_v1 *wp_drm_lease_v1, int32_t leased_fd) +{ + struct direct_wayland_lease *lease = data; + COMP_DEBUG(lease->w->base.base.c, "Lease granted"); + + lease->leased_fd = leased_fd; +} + +static void +_lease_finished(void *data, struct wp_drm_lease_v1 *wp_drm_lease_v1) +{ + struct direct_wayland_lease *lease = data; + if (lease->leased_fd >= 0) { + close(lease->leased_fd); + lease->leased_fd = -1; + } + + COMP_DEBUG(lease->w->base.base.c, "Lease has been terminated"); + lease->finished = true; +} + +static const struct wp_drm_lease_v1_listener lease_listener = { + .lease_fd = _lease_fd, + .finished = _lease_finished, +}; + +static VkResult +comp_window_direct_wayland_create_surface(struct comp_window_direct_wayland *w, + VkSurfaceKHR *surface, + uint32_t width, + uint32_t height) +{ + assert(!w->lease); + + struct vk_bundle *vk = get_vk(w); + VkDisplayKHR _display = VK_NULL_HANDLE; + VkResult ret = VK_ERROR_INCOMPATIBLE_DISPLAY_KHR; + + /* TODO: Choose the connector with an environment variable or from `ct->c->settings.display` */ + + /* Pick the first connector available */ + struct direct_wayland_lease_device *dev = w->devices; + struct direct_wayland_lease_connector *conn = NULL; + while (dev) { + if (dev->connectors) { + conn = dev->connectors; + + COMP_INFO(w->base.base.c, "Using DRM node %s", dev->path); + COMP_INFO(w->base.base.c, "Connector id %d %s (%s)", conn->id, conn->name, conn->description); + break; + } + dev = dev->next; + } + + if (!conn) { + COMP_ERROR(w->base.base.c, "Attempted to create wayland direct surface with no connectors"); + return VK_ERROR_INITIALIZATION_FAILED; + } + + ret = vk->vkGetDrmDisplayEXT(vk->physical_device, dev->drm_fd, conn->id, &_display); + if (ret != VK_SUCCESS) { + COMP_ERROR(w->base.base.c, "vkGetDrmDisplayEXT failed: %s", vk_result_string(ret)); + return ret; + } + + struct wp_drm_lease_request_v1 *request = wp_drm_lease_device_v1_create_lease_request(dev->device); + if (!request) { + COMP_ERROR(w->base.base.c, "Failed to create lease request"); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + + wp_drm_lease_request_v1_request_connector(request, conn->connector); + + struct direct_wayland_lease *lease = calloc(1, sizeof(struct direct_wayland_lease)); + lease->w = w; + lease->leased_fd = -1; + lease->finished = false; + lease->lease = wp_drm_lease_request_v1_submit(request); + + w->lease = lease; + + wp_drm_lease_v1_add_listener(lease->lease, &lease_listener, lease); + + while (lease->leased_fd < 0 && !lease->finished) { + if (wl_display_dispatch(w->display) == -1) { + COMP_ERROR(w->base.base.c, "wl_display roundtrip failed"); + return VK_ERROR_UNKNOWN; + } + } + + if (lease->finished) { + COMP_ERROR(w->base.base.c, "Failed to lease connector"); + return VK_ERROR_UNKNOWN; + } + + ret = vk->vkAcquireDrmDisplayEXT(vk->physical_device, lease->leased_fd, _display); + if (ret != VK_SUCCESS) { + COMP_ERROR(w->base.base.c, "vkAcquireDrmDisplayEXT failed: %s", vk_result_string(ret)); + return ret; + } + + ret = comp_window_direct_create_surface(&w->base, _display, width, height); + if (ret != VK_SUCCESS) { + COMP_ERROR(w->base.base.c, "Failed to create surface: %s", vk_result_string(ret)); + } + + return ret; +} + +static bool +comp_window_direct_wayland_init_swapchain(struct comp_target *w, uint32_t width, uint32_t height) +{ + struct comp_window_direct_wayland *w_wayland = (struct comp_window_direct_wayland *)w; + VkResult ret; + + ret = comp_window_direct_wayland_create_surface(w_wayland, &w_wayland->base.surface.handle, width, height); + if (ret != VK_SUCCESS) { + COMP_ERROR(w->c, "Failed to create surface!"); + return false; + } + + return true; +} + +static void +comp_window_direct_wayland_flush(struct comp_target *w) +{ + struct comp_window_direct_wayland *w_wayland = (struct comp_window_direct_wayland *)w; + + while (wl_display_prepare_read(w_wayland->display) != 0) + wl_display_dispatch_pending(w_wayland->display); + + if (wl_display_flush(w_wayland->display) < 0 && errno != EAGAIN) { + wl_display_cancel_read(w_wayland->display); + return; + } + + struct pollfd fds[] = { + { + .fd = wl_display_get_fd(w_wayland->display), + .events = POLLIN, + }, + }; + + if (poll(fds, 1, 0) > 0) { + wl_display_read_events(w_wayland->display); + wl_display_dispatch_pending(w_wayland->display); + } else { + wl_display_cancel_read(w_wayland->display); + } +} + +static void +_lease_connector_name(void *data, struct wp_drm_lease_connector_v1 *wp_drm_lease_connector_v1, const char *name) +{ + struct direct_wayland_lease_connector *conn = data; + conn->name = strdup(name); +} + +static void +_lease_connector_description(void *data, + struct wp_drm_lease_connector_v1 *wp_drm_lease_connector_v1, + const char *description) +{ + struct direct_wayland_lease_connector *conn = data; + if (conn->description) { + free(conn->description); + } + conn->description = strdup(description); +} + +static void +_lease_connector_connector_id(void *data, + struct wp_drm_lease_connector_v1 *wp_drm_lease_connector_v1, + uint32_t connector_id) +{ + struct direct_wayland_lease_connector *conn = data; + conn->id = connector_id; +} + +static void +_lease_connector_done(void *data, struct wp_drm_lease_connector_v1 *wp_drm_lease_connector_v1) +{ + struct direct_wayland_lease_connector *conn = data; + COMP_DEBUG(conn->w->base.base.c, "[%s] connector %s (%s) id: %d", conn->dev->path, conn->name, + conn->description, conn->id); +} + +static void +_lease_connector_withdrawn(void *data, struct wp_drm_lease_connector_v1 *wp_drm_lease_connector_v1) +{ + struct direct_wayland_lease_connector *conn = data; + COMP_ERROR(conn->w->base.base.c, "Connector %s has been withdrawn by the compositor", conn->name); + + /* TODO: handle graceful shutdown, remove the connector from the device */ +} + +static const struct wp_drm_lease_connector_v1_listener lease_connector_listener = { + .name = _lease_connector_name, + .description = _lease_connector_description, + .connector_id = _lease_connector_connector_id, + .done = _lease_connector_done, + .withdrawn = _lease_connector_withdrawn, +}; + +static void +_drm_lease_device_drm_fd(void *data, struct wp_drm_lease_device_v1 *drm_lease_device, int fd) +{ + struct direct_wayland_lease_device *dev = data; + dev->drm_fd = fd; + dev->path = drmGetDeviceNameFromFd2(fd); + COMP_DEBUG(dev->w->base.base.c, "Available DRM lease device: %s", dev->path); +} + +static void +_drm_lease_device_connector(void *data, + struct wp_drm_lease_device_v1 *wp_drm_lease_device_v1, + struct wp_drm_lease_connector_v1 *wp_drm_lease_connector_v1) +{ + struct direct_wayland_lease_device *dev = data; + + struct direct_wayland_lease_connector *conn = calloc(1, sizeof(struct direct_wayland_lease_connector)); + conn->connector = wp_drm_lease_connector_v1; + conn->dev = dev; + conn->w = dev->w; + wp_drm_lease_connector_v1_add_listener(conn->connector, &lease_connector_listener, conn); + + conn->next = dev->connectors; + dev->connectors = conn; +} + +static void +_drm_lease_device_done(void *data, struct wp_drm_lease_device_v1 *wp_drm_lease_device_v1) +{ + struct direct_wayland_lease_device *dev = data; + dev->done = true; +} + +static void +_drm_lease_device_released(void *data, struct wp_drm_lease_device_v1 *wp_drm_lease_device_v1) +{ + struct direct_wayland_lease_device *dev = data; + COMP_ERROR(dev->w->base.base.c, "Releasing lease device %s", dev->path); + direct_wayland_lease_device_destroy(dev); +} + +static const struct wp_drm_lease_device_v1_listener drm_lease_device_listener = { + .drm_fd = _drm_lease_device_drm_fd, + .connector = _drm_lease_device_connector, + .done = _drm_lease_device_done, + .released = _drm_lease_device_released, +}; + +static void +_registry_global_remove_cb(void *data, struct wl_registry *registry, uint32_t name) +{} + +static void +_registry_global_cb(void *data, struct wl_registry *registry, uint32_t name, const char *interface, uint32_t version) +{ + struct comp_window_direct_wayland *w = data; + if (strcmp(interface, wp_drm_lease_device_v1_interface.name) == 0) { + struct direct_wayland_lease_device *dev = calloc(1, sizeof(struct direct_wayland_lease_device)); + dev->device = (struct wp_drm_lease_device_v1 *)wl_registry_bind(registry, name, + &wp_drm_lease_device_v1_interface, 1); + dev->w = w; + dev->drm_fd = -1; + wp_drm_lease_device_v1_add_listener(dev->device, &drm_lease_device_listener, dev); + + dev->next = w->devices; + w->devices = dev; + } +} + +static const struct wl_registry_listener registry_listener = { + .global = _registry_global_cb, + .global_remove = _registry_global_remove_cb, +}; + +static bool +comp_window_direct_wayland_init(struct comp_target *w) +{ + struct comp_window_direct_wayland *w_wayland = (struct comp_window_direct_wayland *)w; + + w_wayland->display = wl_display_connect(NULL); + if (!w_wayland->display) { + COMP_ERROR(w->c, "Failed to connect to Wayland display"); + return false; + } + + struct wl_registry *registry = wl_display_get_registry(w_wayland->display); + wl_registry_add_listener(registry, ®istry_listener, w_wayland); + wl_display_roundtrip(w_wayland->display); + wl_registry_destroy(registry); + + if (!w_wayland->devices) { + COMP_ERROR(w->c, "Compositor is missing drm-lease support"); + return false; + } + + struct direct_wayland_lease_device *dev = w_wayland->devices; + while (dev) { + if (!dev->done) { + wl_display_dispatch(w_wayland->display); + continue; + } + dev = dev->next; + } + + return true; +} + +static void +_update_window_title(struct comp_target *ct, const char *title) +{ + /* Not required in direct mode */ +} + +struct comp_target * +comp_window_direct_wayland_create(struct comp_compositor *c) +{ + struct comp_window_direct_wayland *w = U_TYPED_CALLOC(struct comp_window_direct_wayland); + + comp_target_swapchain_init_and_set_fnptrs(&w->base, COMP_TARGET_FORCE_FAKE_DISPLAY_TIMING); + + w->base.base.name = "wayland-direct"; + w->base.base.destroy = comp_window_direct_wayland_destroy; + w->base.base.flush = comp_window_direct_wayland_flush; + w->base.base.init_pre_vulkan = comp_window_direct_wayland_init; + w->base.base.init_post_vulkan = comp_window_direct_wayland_init_swapchain; + w->base.base.set_title = _update_window_title; + w->base.base.c = c; + + return &w->base.base; +} diff --git a/src/xrt/compositor/main/comp_window_mswin.c b/src/xrt/compositor/main/comp_window_mswin.c index ae27e233d..88fc763a1 100644 --- a/src/xrt/compositor/main/comp_window_mswin.c +++ b/src/xrt/compositor/main/comp_window_mswin.c @@ -1,8 +1,9 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file - * @brief Wayland window code. + * @brief Microsoft Windows window code. + * @author Ryan Pavlik * @author Lubosz Sarnecki * @author Jakob Bornecrantz * @ingroup comp_main @@ -35,9 +36,11 @@ struct comp_window_mswin bool fullscreen_requested; + bool should_exit; }; static WCHAR szWindowClass[] = L"Monado"; +static WCHAR szWindowData[] = L"MonadoWindow"; /* * @@ -45,23 +48,30 @@ static WCHAR szWindowClass[] = L"Monado"; * */ +static void +draw_window(HWND hWnd, struct comp_window_mswin *cwm) +{ + ValidateRect(hWnd, NULL); +} + static LRESULT CALLBACK WndProc(HWND hWnd, unsigned int message, WPARAM wParam, LPARAM lParam) { + struct comp_window_mswin *cwm = GetPropW(hWnd, szWindowData); + if (!cwm) { + // This is before we've set up our window, or for some other helper window... + // We might want to handle messages differently in here. + return DefWindowProcW(hWnd, message, wParam, lParam); + } switch (message) { - case WM_PAINT: { - // paint the main window - PAINTSTRUCT ps; - HDC hdc = BeginPaint(hWnd, &ps); - // TODO: Add any drawing code that uses hdc here... - EndPaint(hWnd, &ps); - } break; + case WM_PAINT: draw_window(hWnd, cwm); return 0; + case WM_CLOSE: cwm->should_exit = true; return 0; case WM_DESTROY: // Post a quit message and return. - //! @todo set quit flag + cwm->should_exit = true; PostQuitMessage(0); - break; - default: return DefWindowProc(hWnd, message, wParam, lParam); + return 0; + default: return DefWindowProcW(hWnd, message, wParam, lParam); } return 0; } @@ -126,7 +136,7 @@ comp_window_mswin_init_swapchain(struct comp_target *ct, uint32_t width, uint32_ ret = comp_window_mswin_create_surface(cwm, &cwm->base.surface.handle); if (ret != VK_SUCCESS) { - COMP_ERROR(ct->c, "Failed to create surface!"); + COMP_ERROR(ct->c, "Failed to create surface '%s'!", vk_result_string(ret)); return false; } @@ -140,6 +150,12 @@ static void comp_window_mswin_flush(struct comp_target *ct) { struct comp_window_mswin *cwm = (struct comp_window_mswin *)ct; + // force handling messages. + MSG msg; + while (PeekMessageW(&msg, cwm->window, 0, 0, PM_REMOVE)) { + TranslateMessage(&msg); + DispatchMessageW(&msg); + } } @@ -171,6 +187,9 @@ comp_window_mswin_init(struct comp_target *ct) cwm->window = CreateWindowW(szWindowClass, L"Monado (Windowed)", WS_OVERLAPPEDWINDOW, CW_USEDEFAULT, 0, CW_USEDEFAULT, 0, NULL, NULL, cwm->instance, NULL); + SetPropW(cwm->window, szWindowData, cwm); + ShowWindow(cwm->window, SW_SHOWDEFAULT); + UpdateWindow(cwm->window); return true; } @@ -189,7 +208,8 @@ comp_window_mswin_create(struct comp_compositor *c) { struct comp_window_mswin *w = U_TYPED_CALLOC(struct comp_window_mswin); - comp_target_swapchain_init_set_fnptrs(&w->base); + // The display timing code hasn't been tested on Windows and may be broken. + comp_target_swapchain_init_and_set_fnptrs(&w->base, COMP_TARGET_FORCE_FAKE_DISPLAY_TIMING); w->base.base.name = "MS Windows"; w->base.base.destroy = comp_window_mswin_destroy; diff --git a/src/xrt/compositor/main/comp_window_vk_display.c b/src/xrt/compositor/main/comp_window_vk_display.c index 52058cb5f..6ac900372 100644 --- a/src/xrt/compositor/main/comp_window_vk_display.c +++ b/src/xrt/compositor/main/comp_window_vk_display.c @@ -84,7 +84,8 @@ comp_window_vk_display_create(struct comp_compositor *c) { struct comp_window_vk_display *w = U_TYPED_CALLOC(struct comp_window_vk_display); - comp_target_swapchain_init_set_fnptrs(&w->base); + // The display timing code hasn't been tested on vk display and may be broken. + comp_target_swapchain_init_and_set_fnptrs(&w->base, COMP_TARGET_FORCE_FAKE_DISPLAY_TIMING); w->base.base.name = "VkDisplayKHR"; w->base.base.destroy = comp_window_vk_display_destroy; @@ -151,18 +152,16 @@ static bool comp_window_vk_display_init(struct comp_target *ct) { struct comp_window_vk_display *w_direct = (struct comp_window_vk_display *)ct; + struct vk_bundle *vk = &ct->c->vk; // Sanity check. - if (ct->c->vk.instance == VK_NULL_HANDLE) { + if (vk->instance == VK_NULL_HANDLE) { COMP_ERROR(ct->c, "Vulkan not initialized before vk display init!"); return false; } - struct vk_bundle comp_vk = ct->c->vk; - uint32_t display_count; - if (comp_vk.vkGetPhysicalDeviceDisplayPropertiesKHR(comp_vk.physical_device, &display_count, NULL) != - VK_SUCCESS) { + if (vk->vkGetPhysicalDeviceDisplayPropertiesKHR(vk->physical_device, &display_count, NULL) != VK_SUCCESS) { COMP_ERROR(ct->c, "Failed to get vulkan display count"); return false; } @@ -174,8 +173,8 @@ comp_window_vk_display_init(struct comp_target *ct) struct VkDisplayPropertiesKHR *display_props = U_TYPED_ARRAY_CALLOC(VkDisplayPropertiesKHR, display_count); - if (display_props && comp_vk.vkGetPhysicalDeviceDisplayPropertiesKHR(comp_vk.physical_device, &display_count, - display_props) != VK_SUCCESS) { + if (display_props && vk->vkGetPhysicalDeviceDisplayPropertiesKHR(vk->physical_device, &display_count, + display_props) != VK_SUCCESS) { COMP_ERROR(ct->c, "Failed to get display properties"); free(display_props); return false; diff --git a/src/xrt/compositor/main/comp_window_wayland.c b/src/xrt/compositor/main/comp_window_wayland.c index 771b5a6ed..c22a9959b 100644 --- a/src/xrt/compositor/main/comp_window_wayland.c +++ b/src/xrt/compositor/main/comp_window_wayland.c @@ -102,7 +102,8 @@ comp_window_wayland_create(struct comp_compositor *c) { struct comp_window_wayland *w = U_TYPED_CALLOC(struct comp_window_wayland); - comp_target_swapchain_init_set_fnptrs(&w->base); + // The display timing code hasn't been tested on Wayland and may be broken. + comp_target_swapchain_init_and_set_fnptrs(&w->base, COMP_TARGET_FORCE_FAKE_DISPLAY_TIMING); w->base.base.name = "wayland"; w->base.base.destroy = comp_window_wayland_destroy; diff --git a/src/xrt/compositor/main/comp_window_xcb.c b/src/xrt/compositor/main/comp_window_xcb.c index 5ec992cbf..07fb91fd9 100644 --- a/src/xrt/compositor/main/comp_window_xcb.c +++ b/src/xrt/compositor/main/comp_window_xcb.c @@ -128,7 +128,11 @@ comp_window_xcb_create(struct comp_compositor *c) { struct comp_window_xcb *w = U_TYPED_CALLOC(struct comp_window_xcb); - comp_target_swapchain_init_set_fnptrs(&w->base); + /* + * The display timing code has been tested on XCB, + * and is know to be broken when using VK_PRESENT_MODE_IMMEDIATE_KHR. + */ + comp_target_swapchain_init_and_set_fnptrs(&w->base, COMP_TARGET_FORCE_FAKE_DISPLAY_TIMING); w->base.base.name = "xcb"; w->base.base.destroy = comp_window_xcb_destroy; diff --git a/src/xrt/compositor/meson.build b/src/xrt/compositor/meson.build index 3ff8b758a..6d2b96065 100644 --- a/src/xrt/compositor/meson.build +++ b/src/xrt/compositor/meson.build @@ -1,4 +1,4 @@ -# Copyright 2019-2020, Collabora, Ltd. +# Copyright 2019-2021, Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 subdir('shaders') @@ -22,13 +22,19 @@ compositor_srcs = [ 'main/comp_settings.h', 'main/comp_shaders.c', 'main/comp_swapchain.c', + 'main/comp_sync.c', 'main/comp_target.h', 'main/comp_target_swapchain.c', 'main/comp_target_swapchain.h', 'main/comp_window.h', 'main/comp_layer_renderer.c', 'main/comp_layer.c', + 'multi/comp_multi_compositor.c', + 'multi/comp_multi_interface.h', + 'multi/comp_multi_private.h', + 'multi/comp_multi_system.c', 'render/comp_buffer.c', + 'render/comp_compute.c', 'render/comp_render.h', 'render/comp_rendering.c', 'render/comp_resources.c', @@ -36,22 +42,25 @@ compositor_srcs = [ compile_args = [] -if build_vk_khr_display - compositor_srcs += ['main/comp_window_vk_display.c'] -endif - if build_xcb compositor_srcs += ['main/comp_window_xcb.c'] compositor_deps += [xcb] endif +if build_vk_khr_display + compositor_srcs += ['main/comp_window_vk_display.c'] +endif + if build_xcb_xrandr_direct - compositor_srcs += ['main/comp_window_direct.c', - 'main/comp_window_direct_randr.c', + compositor_srcs += ['main/comp_window_direct_randr.c', 'main/comp_window_direct_nvidia.c'] compositor_deps += [xcb_randr, x11_xcb] endif +if build_vk_khr_display or build_xcb_xrandr_direct + compositor_srcs += ['main/comp_window_direct.c'] +endif + if build_opengl or build_opengles compositor_srcs += [ 'client/comp_gl_client.c', @@ -93,7 +102,8 @@ endif if build_egl compositor_srcs += [ - 'client/comp_egl_glue.c', + 'client/comp_egl_client.c', + 'client/comp_egl_client.h', ] compositor_deps += [egl] endif @@ -104,20 +114,27 @@ if build_wayland wl_protocol_dir = wayland_protos.get_pkgconfig_variable('pkgdatadir') protocols = [ - [wl_protocol_dir, 'stable/xdg-shell/xdg-shell.xml'], + wl_protocol_dir / 'stable/xdg-shell/xdg-shell.xml', ] - foreach p : protocols - xml = join_paths(p) + if build_wayland_direct + protocols += wl_protocol_dir / 'staging/drm-lease/drm-lease-v1.xml' + compositor_srcs += 'main/comp_window_direct_wayland.c' + compositor_deps += drm + endif + + compositor_srcs += 'main/comp_window_wayland.c' + + foreach path : protocols wl_protos_src += custom_target( - xml.underscorify() + '_c', - input: xml, + path.underscorify() + '_c', + input: path, output: '@BASENAME@-protocol.c', command: [wayland_scanner, 'private-code', '@INPUT@', '@OUTPUT@'], ) wl_protos_headers += custom_target( - xml.underscorify() + '_client_h', - input: xml, + path.underscorify() + '_client_h', + input: path, output: '@BASENAME@-client-protocol.h', command: [wayland_scanner, 'client-header', '@INPUT@', '@OUTPUT@'], ) @@ -134,8 +151,8 @@ if build_wayland sources: wl_protos_headers, ) - compositor_srcs += ['main/comp_window_wayland.c'] compositor_deps += [wayland, wl_protos] + endif lib_comp = static_library( diff --git a/src/xrt/compositor/multi/comp_multi_compositor.c b/src/xrt/compositor/multi/comp_multi_compositor.c new file mode 100644 index 000000000..83431b559 --- /dev/null +++ b/src/xrt/compositor/multi/comp_multi_compositor.c @@ -0,0 +1,650 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Multi client wrapper compositor. + * @author Pete Black + * @author Jakob Bornecrantz + * @ingroup comp_multi + */ + +#include "xrt/xrt_gfx_native.h" + +#include "os/os_time.h" + +#include "util/u_var.h" +#include "util/u_misc.h" +#include "util/u_time.h" +#include "util/u_debug.h" +#include "util/u_handles.h" +#include "util/u_trace_marker.h" +#include "util/u_distortion_mesh.h" + +#include "multi/comp_multi_private.h" + +#include +#include +#include +#include +#include +#include + +#ifdef XRT_GRAPHICS_SYNC_HANDLE_IS_FD +#include +#endif + + +/* + * + * Slot management functions. + * + */ + +static void +slot_clear(struct multi_layer_slot *slot) +{ + for (size_t i = 0; i < slot->num_layers; i++) { + for (size_t k = 0; k < ARRAY_SIZE(slot->layers[i].xscs); k++) { + xrt_swapchain_reference(&slot->layers[i].xscs[k], NULL); + } + } + + U_ZERO(slot); +} + +static void +slot_move_and_clear(struct multi_layer_slot *dst, struct multi_layer_slot *src) +{ + slot_clear(dst); + + // All references are kept. + *dst = *src; + + U_ZERO(src); +} + + +/* + * + * Event management functions. + * + */ + +void +multi_compositor_push_event(struct multi_compositor *mc, const union xrt_compositor_event *xce) +{ + struct multi_event *me = U_TYPED_CALLOC(struct multi_event); + me->xce = *xce; + + os_mutex_lock(&mc->event.mutex); + + // Find the last slot. + struct multi_event **slot = &mc->event.next; + while (*slot != NULL) { + slot = &(*slot)->next; + } + + *slot = me; + + os_mutex_unlock(&mc->event.mutex); +} + +static void +pop_event(struct multi_compositor *mc, union xrt_compositor_event *out_xce) +{ + out_xce->type = XRT_COMPOSITOR_EVENT_NONE; + + os_mutex_lock(&mc->event.mutex); + + if (mc->event.next != NULL) { + struct multi_event *me = mc->event.next; + + *out_xce = me->xce; + mc->event.next = me->next; + free(me); + } + + os_mutex_unlock(&mc->event.mutex); +} + +static void +drain_events(struct multi_compositor *mc) +{ + union xrt_compositor_event xce; + do { + pop_event(mc, &xce); + } while (xce.type != XRT_COMPOSITOR_EVENT_NONE); +} + + +/* + * + * Compositor functions. + * + */ + +static xrt_result_t +multi_compositor_create_swapchain(struct xrt_compositor *xc, + const struct xrt_swapchain_create_info *info, + struct xrt_swapchain **out_xsc) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + + return xrt_comp_create_swapchain(&mc->msc->xcn->base, info, out_xsc); +} + +static xrt_result_t +multi_compositor_import_swapchain(struct xrt_compositor *xc, + const struct xrt_swapchain_create_info *info, + struct xrt_image_native *native_images, + uint32_t num_images, + struct xrt_swapchain **out_xsc) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + + return xrt_comp_import_swapchain(&mc->msc->xcn->base, info, native_images, num_images, out_xsc); +} + +static xrt_result_t +multi_compositor_import_fence(struct xrt_compositor *xc, + xrt_graphics_sync_handle_t handle, + struct xrt_compositor_fence **out_xcf) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + + return xrt_comp_import_fence(&mc->msc->xcn->base, handle, out_xcf); +} + +static xrt_result_t +multi_compositor_begin_session(struct xrt_compositor *xc, enum xrt_view_type type) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + (void)mc; + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_end_session(struct xrt_compositor *xc) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + (void)mc; + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_predict_frame(struct xrt_compositor *xc, + int64_t *out_frame_id, + uint64_t *out_wake_time_ns, + uint64_t *out_predicted_gpu_time_ns, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + + os_mutex_lock(&mc->msc->list_and_timing_lock); + + u_rt_predict( // + mc->urt, // + out_frame_id, // + out_wake_time_ns, // + out_predicted_display_time_ns, // + out_predicted_display_period_ns); // + + os_mutex_unlock(&mc->msc->list_and_timing_lock); + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_mark_frame(struct xrt_compositor *xc, + int64_t frame_id, + enum xrt_compositor_frame_point point, + uint64_t when_ns) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + + switch (point) { + case XRT_COMPOSITOR_FRAME_POINT_WOKE: + os_mutex_lock(&mc->msc->list_and_timing_lock); + uint64_t now_ns = os_monotonic_get_ns(); + u_rt_mark_point(mc->urt, frame_id, U_TIMING_POINT_WAKE_UP, now_ns); + os_mutex_unlock(&mc->msc->list_and_timing_lock); + break; + default: assert(false); + } + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_wait_frame(struct xrt_compositor *xc, + int64_t *out_frame_id, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + + int64_t frame_id = -1; + uint64_t wake_up_time_ns = 0; + uint64_t predicted_gpu_time_ns = 0; + + xrt_comp_predict_frame( // + xc, // + &frame_id, // + &wake_up_time_ns, // + &predicted_gpu_time_ns, // + out_predicted_display_time_ns, // + out_predicted_display_period_ns); // + + uint64_t now_ns = os_monotonic_get_ns(); + if (now_ns < wake_up_time_ns) { + uint32_t delay = (uint32_t)(wake_up_time_ns - now_ns); + os_precise_sleeper_nanosleep(&mc->sleeper, delay); + } + + now_ns = os_monotonic_get_ns(); + + xrt_comp_mark_frame(xc, frame_id, XRT_COMPOSITOR_FRAME_POINT_WOKE, now_ns); + + *out_frame_id = frame_id; + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_begin_frame(struct xrt_compositor *xc, int64_t frame_id) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + + os_mutex_lock(&mc->msc->list_and_timing_lock); + uint64_t now_ns = os_monotonic_get_ns(); + u_rt_mark_point(mc->urt, frame_id, U_TIMING_POINT_BEGIN, now_ns); + os_mutex_unlock(&mc->msc->list_and_timing_lock); + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_discard_frame(struct xrt_compositor *xc, int64_t frame_id) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + + os_mutex_lock(&mc->msc->list_and_timing_lock); + u_rt_mark_discarded(mc->urt, frame_id); + os_mutex_unlock(&mc->msc->list_and_timing_lock); + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_layer_begin(struct xrt_compositor *xc, + int64_t frame_id, + uint64_t display_time_ns, + enum xrt_blend_mode env_blend_mode) +{ + struct multi_compositor *mc = multi_compositor(xc); + + assert(mc->progress.num_layers == 0); + U_ZERO(&mc->progress); + + mc->progress.active = true; + mc->progress.display_time_ns = display_time_ns; + mc->progress.env_blend_mode = env_blend_mode; + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_layer_stereo_projection(struct xrt_compositor *xc, + struct xrt_device *xdev, + struct xrt_swapchain *l_xsc, + struct xrt_swapchain *r_xsc, + const struct xrt_layer_data *data) +{ + struct multi_compositor *mc = multi_compositor(xc); + (void)mc; + + size_t index = mc->progress.num_layers++; + mc->progress.layers[index].xdev = xdev; + xrt_swapchain_reference(&mc->progress.layers[index].xscs[0], l_xsc); + xrt_swapchain_reference(&mc->progress.layers[index].xscs[1], r_xsc); + mc->progress.layers[index].data = *data; + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_layer_stereo_projection_depth(struct xrt_compositor *xc, + struct xrt_device *xdev, + struct xrt_swapchain *l_xsc, + struct xrt_swapchain *r_xsc, + struct xrt_swapchain *l_d_xsc, + struct xrt_swapchain *r_d_xsc, + const struct xrt_layer_data *data) +{ + struct multi_compositor *mc = multi_compositor(xc); + + size_t index = mc->progress.num_layers++; + mc->progress.layers[index].xdev = xdev; + xrt_swapchain_reference(&mc->progress.layers[index].xscs[0], l_xsc); + xrt_swapchain_reference(&mc->progress.layers[index].xscs[1], r_xsc); + xrt_swapchain_reference(&mc->progress.layers[index].xscs[2], l_d_xsc); + xrt_swapchain_reference(&mc->progress.layers[index].xscs[3], r_d_xsc); + mc->progress.layers[index].data = *data; + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_layer_quad(struct xrt_compositor *xc, + struct xrt_device *xdev, + struct xrt_swapchain *xsc, + const struct xrt_layer_data *data) +{ + struct multi_compositor *mc = multi_compositor(xc); + + size_t index = mc->progress.num_layers++; + mc->progress.layers[index].xdev = xdev; + xrt_swapchain_reference(&mc->progress.layers[index].xscs[0], xsc); + mc->progress.layers[index].data = *data; + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_layer_cube(struct xrt_compositor *xc, + struct xrt_device *xdev, + struct xrt_swapchain *xsc, + const struct xrt_layer_data *data) +{ + struct multi_compositor *mc = multi_compositor(xc); + + size_t index = mc->progress.num_layers++; + mc->progress.layers[index].xdev = xdev; + xrt_swapchain_reference(&mc->progress.layers[index].xscs[0], xsc); + mc->progress.layers[index].data = *data; + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_layer_cylinder(struct xrt_compositor *xc, + struct xrt_device *xdev, + struct xrt_swapchain *xsc, + const struct xrt_layer_data *data) +{ + struct multi_compositor *mc = multi_compositor(xc); + + size_t index = mc->progress.num_layers++; + mc->progress.layers[index].xdev = xdev; + xrt_swapchain_reference(&mc->progress.layers[index].xscs[0], xsc); + mc->progress.layers[index].data = *data; + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_layer_equirect1(struct xrt_compositor *xc, + struct xrt_device *xdev, + struct xrt_swapchain *xsc, + const struct xrt_layer_data *data) +{ + struct multi_compositor *mc = multi_compositor(xc); + + size_t index = mc->progress.num_layers++; + mc->progress.layers[index].xdev = xdev; + xrt_swapchain_reference(&mc->progress.layers[index].xscs[0], xsc); + mc->progress.layers[index].data = *data; + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_layer_equirect2(struct xrt_compositor *xc, + struct xrt_device *xdev, + struct xrt_swapchain *xsc, + const struct xrt_layer_data *data) +{ + struct multi_compositor *mc = multi_compositor(xc); + + size_t index = mc->progress.num_layers++; + mc->progress.layers[index].xdev = xdev; + xrt_swapchain_reference(&mc->progress.layers[index].xscs[0], xsc); + mc->progress.layers[index].data = *data; + + return XRT_SUCCESS; +} + +static void +wait_fence(struct xrt_compositor_fence **xcf_ptr) +{ + COMP_TRACE_MARKER(); + xrt_compositor_fence_wait(*xcf_ptr, UINT64_MAX); + xrt_compositor_fence_destroy(xcf_ptr); +} + +static void +wait_for_scheduled_free(struct multi_compositor *mc) +{ + COMP_TRACE_MARKER(); + + os_mutex_lock(&mc->slot_lock); + + // Block here if the scheduled slot is not clear. + while (mc->scheduled.active) { + + // Replace the scheduled frame if it's in the past. + uint64_t now_ns = os_monotonic_get_ns(); + if (mc->scheduled.display_time_ns < now_ns) { + break; + } + + os_mutex_unlock(&mc->slot_lock); + + os_nanosleep(U_TIME_1MS_IN_NS); + + os_mutex_lock(&mc->slot_lock); + } + + slot_move_and_clear(&mc->scheduled, &mc->progress); + + os_mutex_unlock(&mc->slot_lock); +} + +static xrt_result_t +multi_compositor_layer_commit(struct xrt_compositor *xc, int64_t frame_id, xrt_graphics_sync_handle_t sync_handle) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + struct xrt_compositor_fence *xcf = NULL; + + do { + if (!xrt_graphics_sync_handle_is_valid(sync_handle)) { + break; + } + + xrt_result_t xret = xrt_comp_import_fence( // + &mc->msc->xcn->base, // + sync_handle, // + &xcf); // + /*! + * If import_fence succeeded, we have transferred ownership to + * the compositor no need to do anything more. If the call + * failed we need to close the handle. + */ + if (xret == XRT_SUCCESS) { + break; + } + + u_graphics_sync_unref(&sync_handle); + } while (false); // Goto without the labels. + + if (xcf != NULL) { + wait_fence(&xcf); + } + + wait_for_scheduled_free(mc); + + os_mutex_lock(&mc->msc->list_and_timing_lock); + u_rt_mark_delivered(mc->urt, frame_id); + os_mutex_unlock(&mc->msc->list_and_timing_lock); + + return XRT_SUCCESS; +} + +static xrt_result_t +multi_compositor_poll_events(struct xrt_compositor *xc, union xrt_compositor_event *out_xce) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + + pop_event(mc, out_xce); + + return XRT_SUCCESS; +} + +static void +multi_compositor_destroy(struct xrt_compositor *xc) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = multi_compositor(xc); + + os_mutex_lock(&mc->msc->list_and_timing_lock); + + // Remove it from the list of clients. + for (size_t i = 0; i < MULTI_MAX_CLIENTS; i++) { + if (mc->msc->clients[i] == mc) { + mc->msc->clients[i] = NULL; + } + } + + os_mutex_unlock(&mc->msc->list_and_timing_lock); + + drain_events(mc); + + // We are now off the rendering list, clear slots for any swapchains. + slot_clear(&mc->progress); + slot_clear(&mc->scheduled); + slot_clear(&mc->delivered); + + // Does null checking. + u_rt_destroy(&mc->urt); + + os_precise_sleeper_deinit(&mc->sleeper); + + os_mutex_destroy(&mc->slot_lock); + os_mutex_destroy(&mc->event.mutex); + + free(mc); +} + +void +multi_compositor_deliver_any_frames(struct multi_compositor *mc, uint64_t display_time_ns) +{ + os_mutex_lock(&mc->slot_lock); + + if (!mc->scheduled.active) { + os_mutex_unlock(&mc->slot_lock); + return; + } + + if (time_is_greater_then_or_within_half_ms(display_time_ns, mc->scheduled.display_time_ns)) { + slot_move_and_clear(&mc->delivered, &mc->scheduled); + } + + os_mutex_unlock(&mc->slot_lock); +} + +xrt_result_t +multi_compositor_create(struct multi_system_compositor *msc, + const struct xrt_session_info *xsi, + struct xrt_compositor_native **out_xcn) +{ + COMP_TRACE_MARKER(); + + struct multi_compositor *mc = U_TYPED_CALLOC(struct multi_compositor); + + mc->base.base.create_swapchain = multi_compositor_create_swapchain; + mc->base.base.import_swapchain = multi_compositor_import_swapchain; + mc->base.base.import_fence = multi_compositor_import_fence; + mc->base.base.begin_session = multi_compositor_begin_session; + mc->base.base.end_session = multi_compositor_end_session; + mc->base.base.predict_frame = multi_compositor_predict_frame; + mc->base.base.mark_frame = multi_compositor_mark_frame; + mc->base.base.wait_frame = multi_compositor_wait_frame; + mc->base.base.begin_frame = multi_compositor_begin_frame; + mc->base.base.discard_frame = multi_compositor_discard_frame; + mc->base.base.layer_begin = multi_compositor_layer_begin; + mc->base.base.layer_stereo_projection = multi_compositor_layer_stereo_projection; + mc->base.base.layer_stereo_projection_depth = multi_compositor_layer_stereo_projection_depth; + mc->base.base.layer_quad = multi_compositor_layer_quad; + mc->base.base.layer_cube = multi_compositor_layer_cube; + mc->base.base.layer_cylinder = multi_compositor_layer_cylinder; + mc->base.base.layer_equirect1 = multi_compositor_layer_equirect1; + mc->base.base.layer_equirect2 = multi_compositor_layer_equirect2; + mc->base.base.layer_commit = multi_compositor_layer_commit; + mc->base.base.destroy = multi_compositor_destroy; + mc->base.base.poll_events = multi_compositor_poll_events; + mc->msc = msc; + mc->xsi = *xsi; + + os_mutex_init(&mc->event.mutex); + os_mutex_init(&mc->slot_lock); + + // Passthrough our formats from the native compositor to the client. + mc->base.base.info = msc->xcn->base.info; + + // Using in wait frame. + os_precise_sleeper_init(&mc->sleeper); + + // This is safe to do without a lock since we are not on the list yet. + u_rt_create(&mc->urt); + + os_mutex_lock(&msc->list_and_timing_lock); + + // Meh if we have to many clients just ignore it. + for (size_t i = 0; i < MULTI_MAX_CLIENTS; i++) { + if (mc->msc->clients[i] != NULL) { + continue; + } + mc->msc->clients[i] = mc; + break; + } + + u_rt_info( // + mc->urt, // + msc->last_timings.predicted_display_time_ns, // + msc->last_timings.predicted_display_period_ns, // + msc->last_timings.diff_ns); // + + os_mutex_unlock(&msc->list_and_timing_lock); + + *out_xcn = &mc->base; + + return XRT_SUCCESS; +} diff --git a/src/xrt/compositor/multi/comp_multi_interface.h b/src/xrt/compositor/multi/comp_multi_interface.h new file mode 100644 index 000000000..55b24edf3 --- /dev/null +++ b/src/xrt/compositor/multi/comp_multi_interface.h @@ -0,0 +1,28 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Interface for the multi-client layer code. + * @author Jakob Bornecrantz + * @ingroup comp_main + */ + +#pragma once + +#include "xrt/xrt_compositor.h" + + +#ifdef __cplusplus +extern "C" { +#endif + + +xrt_result_t +comp_multi_create_system_compositor(struct xrt_compositor_native *xcn, + const struct xrt_system_compositor_info *xsci, + struct xrt_system_compositor **out_xsysc); + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/compositor/multi/comp_multi_private.h b/src/xrt/compositor/multi/comp_multi_private.h new file mode 100644 index 000000000..eabe6cb8e --- /dev/null +++ b/src/xrt/compositor/multi/comp_multi_private.h @@ -0,0 +1,229 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Multi-client compositor internal structs. + * @author Jakob Bornecrantz + * @ingroup comp_multi + */ + +#pragma once + +#include "xrt/xrt_compiler.h" +#include "xrt/xrt_defines.h" +#include "xrt/xrt_compositor.h" + +#include "os/os_time.h" +#include "os/os_threading.h" + +#include "util/u_timing.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +#define MULTI_MAX_CLIENTS 64 +#define MULTI_MAX_LAYERS 16 + + +/* + * + * Native compositor. + * + */ + +/*! + * Data for a single composition layer. + * + * Similar in function to @ref comp_layer + * + * @ingroup comp_multi + */ +struct multi_layer_entry +{ + /*! + * Device to get pose from. + */ + struct xrt_device *xdev; + + /*! + * Pointers to swapchains. + * + * How many are actually used depends on the value of @p data.type + */ + struct xrt_swapchain *xscs[4]; + + /*! + * All basic (trivially-serializable) data associated with a layer, + * aside from which swapchain(s) are used. + */ + struct xrt_layer_data data; +}; + +/*! + * Render state for a single client, including all layers. + * + * @ingroup comp_multi + */ +struct multi_layer_slot +{ + uint64_t display_time_ns; //!< When should this be shown, @see XrFrameEndInfo::displayTime. + enum xrt_blend_mode env_blend_mode; + uint32_t num_layers; + struct multi_layer_entry layers[MULTI_MAX_LAYERS]; + bool active; +}; + +/*! + * Render state for a single client, including all layers. + * + * @ingroup comp_multi + */ +struct multi_event +{ + struct multi_event *next; + union xrt_compositor_event xce; +}; + +/*! + * A single compositor. + * + * @ingroup comp_multi + */ +struct multi_compositor +{ + struct xrt_compositor_native base; + + // Client info. + struct xrt_session_info xsi; + + //! Owning system compositor. + struct multi_system_compositor *msc; + + //! Only matters for Windows and in process. + struct os_precise_sleeper sleeper; + + struct + { + struct os_mutex mutex; + struct multi_event *next; + } event; + + struct + { + struct + { + bool visible; + bool focused; + } sent; + struct + { + bool visible; + bool focused; + } current; + + int64_t z_order; + } state; + + //! Lock for all of the slots. + struct os_mutex slot_lock; + + /*! + * Currently being transferred or waited on. + * Not protected by the slot lock as it is only touched by the client thread. + */ + struct multi_layer_slot progress; + + //! Scheduled frames for a future timepoint. + struct multi_layer_slot scheduled; + + /*! + * Fully ready to be used. + * Not protected by the slot lock as it is only touched by the main render loop thread. + */ + struct multi_layer_slot delivered; + + struct u_render_timing *urt; +}; + +static inline struct multi_compositor * +multi_compositor(struct xrt_compositor *xc) +{ + return (struct multi_compositor *)xc; +} + +/*! + * Create a multi client wrapper compositor. + * + * @ingroup comp_multi + */ +xrt_result_t +multi_compositor_create(struct multi_system_compositor *msc, + const struct xrt_session_info *xsi, + struct xrt_compositor_native **out_xcn); + +/*! + * Push a event to be delivered to the client. + * + * @ingroup comp_multi + */ +void +multi_compositor_push_event(struct multi_compositor *mc, const union xrt_compositor_event *xce); + +/*! + * Deliver any scheduled frames at that is to be display at or after the given @p display_time_ns. Called by the render + * thread and copies data from multi_compositor::scheduled to multi_compositor::delivered while holding the slot_lock. + * + * @ingroup comp_multi + */ +void +multi_compositor_deliver_any_frames(struct multi_compositor *mc, uint64_t display_time_ns); + + +/* + * + * System compositor. + * + */ + +struct multi_system_compositor +{ + struct xrt_system_compositor base; + + //! Extra functions to handle multi client. + struct xrt_multi_compositor_control xmcc; + + //! Real native compositor. + struct xrt_compositor_native *xcn; + + //! Render loop thread. + struct os_thread_helper oth; + + /*! + * This mutex protects the list of client compositor + * and the rendering timings on it. + */ + struct os_mutex list_and_timing_lock; + + struct + { + uint64_t predicted_display_time_ns; + uint64_t predicted_display_period_ns; + uint64_t diff_ns; + } last_timings; + + struct multi_compositor *clients[MULTI_MAX_CLIENTS]; +}; + +static inline struct multi_system_compositor * +multi_system_compositor(struct xrt_system_compositor *xsc) +{ + return (struct multi_system_compositor *)xsc; +} + + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/compositor/multi/comp_multi_system.c b/src/xrt/compositor/multi/comp_multi_system.c new file mode 100644 index 000000000..5fe7157ae --- /dev/null +++ b/src/xrt/compositor/multi/comp_multi_system.c @@ -0,0 +1,539 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Multi client wrapper compositor. + * @author Pete Black + * @author Jakob Bornecrantz + * @ingroup comp_multi + */ + +#include "xrt/xrt_gfx_native.h" + +#include "os/os_time.h" + +#include "util/u_var.h" +#include "util/u_misc.h" +#include "util/u_time.h" +#include "util/u_debug.h" +#include "util/u_trace_marker.h" +#include "util/u_distortion_mesh.h" + +#include "multi/comp_multi_private.h" + +#include +#include +#include +#include +#include +#include + +#ifdef XRT_GRAPHICS_SYNC_HANDLE_IS_FD +#include +#endif + + +/* + * + * Render thread. + * + */ + +static void +do_projection_layer(struct xrt_compositor *xc, struct multi_compositor *mc, struct multi_layer_entry *layer, uint32_t i) +{ + struct xrt_device *xdev = layer->xdev; + struct xrt_swapchain *l_xcs = layer->xscs[0]; + struct xrt_swapchain *r_xcs = layer->xscs[1]; + + if (l_xcs == NULL || r_xcs == NULL) { + U_LOG_E("Invalid swap chain for projection layer #%u!", i); + return; + } + + if (xdev == NULL) { + U_LOG_E("Invalid xdev for projection layer #%u!", i); + return; + } + + // Cast away + struct xrt_layer_data *data = (struct xrt_layer_data *)&layer->data; + + xrt_comp_layer_stereo_projection(xc, xdev, l_xcs, r_xcs, data); +} + +static void +do_projection_layer_depth(struct xrt_compositor *xc, + struct multi_compositor *mc, + struct multi_layer_entry *layer, + uint32_t i) +{ + struct xrt_device *xdev = layer->xdev; + struct xrt_swapchain *l_xcs = layer->xscs[0]; + struct xrt_swapchain *r_xcs = layer->xscs[1]; + struct xrt_swapchain *l_d_xcs = layer->xscs[2]; + struct xrt_swapchain *r_d_xcs = layer->xscs[2]; + + if (l_xcs == NULL || r_xcs == NULL || l_d_xcs == NULL || r_d_xcs == NULL) { + U_LOG_E("Invalid swap chain for projection layer #%u!", i); + return; + } + + if (xdev == NULL) { + U_LOG_E("Invalid xdev for projection layer #%u!", i); + return; + } + + // Cast away + struct xrt_layer_data *data = (struct xrt_layer_data *)&layer->data; + + xrt_comp_layer_stereo_projection_depth(xc, xdev, l_xcs, r_xcs, l_d_xcs, r_d_xcs, data); +} + +static bool +do_single(struct xrt_compositor *xc, + struct multi_compositor *mc, + struct multi_layer_entry *layer, + uint32_t i, + const char *name, + struct xrt_device **out_xdev, + struct xrt_swapchain **out_xcs, + struct xrt_layer_data **out_data) +{ + struct xrt_device *xdev = layer->xdev; + struct xrt_swapchain *xcs = layer->xscs[0]; + + if (xcs == NULL) { + U_LOG_E("Invalid swapchain for layer #%u '%s'!", i, name); + return false; + } + + if (xdev == NULL) { + U_LOG_E("Invalid xdev for layer #%u '%s'!", i, name); + return false; + } + + // Cast away + struct xrt_layer_data *data = (struct xrt_layer_data *)&layer->data; + + *out_xdev = xdev; + *out_xcs = xcs; + *out_data = data; + + return true; +} + +static void +do_quad_layer(struct xrt_compositor *xc, struct multi_compositor *mc, struct multi_layer_entry *layer, uint32_t i) +{ + struct xrt_device *xdev = NULL; + struct xrt_swapchain *xcs = NULL; + struct xrt_layer_data *data = NULL; + + if (!do_single(xc, mc, layer, i, "quad", &xdev, &xcs, &data)) { + return; + } + + xrt_comp_layer_quad(xc, xdev, xcs, data); +} + +static void +do_cube_layer(struct xrt_compositor *xc, struct multi_compositor *mc, struct multi_layer_entry *layer, uint32_t i) +{ + struct xrt_device *xdev = NULL; + struct xrt_swapchain *xcs = NULL; + struct xrt_layer_data *data = NULL; + + if (!do_single(xc, mc, layer, i, "cube", &xdev, &xcs, &data)) { + return; + } + + xrt_comp_layer_cube(xc, xdev, xcs, data); +} + +static void +do_cylinder_layer(struct xrt_compositor *xc, struct multi_compositor *mc, struct multi_layer_entry *layer, uint32_t i) +{ + struct xrt_device *xdev = NULL; + struct xrt_swapchain *xcs = NULL; + struct xrt_layer_data *data = NULL; + + if (!do_single(xc, mc, layer, i, "cylinder", &xdev, &xcs, &data)) { + return; + } + + xrt_comp_layer_cylinder(xc, xdev, xcs, data); +} + +static void +do_equirect1_layer(struct xrt_compositor *xc, struct multi_compositor *mc, struct multi_layer_entry *layer, uint32_t i) +{ + struct xrt_device *xdev = NULL; + struct xrt_swapchain *xcs = NULL; + struct xrt_layer_data *data = NULL; + + if (!do_single(xc, mc, layer, i, "equirect1", &xdev, &xcs, &data)) { + return; + } + + xrt_comp_layer_equirect1(xc, xdev, xcs, data); +} + +static void +do_equirect2_layer(struct xrt_compositor *xc, struct multi_compositor *mc, struct multi_layer_entry *layer, uint32_t i) +{ + struct xrt_device *xdev = NULL; + struct xrt_swapchain *xcs = NULL; + struct xrt_layer_data *data = NULL; + + if (!do_single(xc, mc, layer, i, "equirect2", &xdev, &xcs, &data)) { + return; + } + + xrt_comp_layer_equirect2(xc, xdev, xcs, data); +} + +static int +overlay_sort_func(const void *a, const void *b) +{ + struct multi_compositor *mc_a = *(struct multi_compositor **)a; + struct multi_compositor *mc_b = *(struct multi_compositor **)b; + + if (mc_a->state.z_order < mc_b->state.z_order) { + return -1; + } + + if (mc_a->state.z_order > mc_b->state.z_order) { + return 1; + } + + return 0; +} + +static void +log_frame_time_diff(uint64_t frame_time_ns, uint64_t display_time_ns) +{ + int64_t diff_ns = (int64_t)frame_time_ns - (int64_t)display_time_ns; + bool late = false; + if (diff_ns < 0) { + diff_ns = -diff_ns; + late = true; + } + + U_LOG_W("Frame %s by %.2fms!", late ? "late" : "early", time_ns_to_ms_f(diff_ns)); +} + +static void +transfer_layers_locked(struct multi_system_compositor *msc, uint64_t display_time_ns) +{ + COMP_TRACE_MARKER(); + + struct xrt_compositor *xc = &msc->xcn->base; + + struct multi_compositor *array[MULTI_MAX_CLIENTS] = {0}; + + size_t count = 0; + for (size_t k = 0; k < ARRAY_SIZE(array); k++) { + if (msc->clients[k] == NULL) { + continue; + } + + array[count++] = msc->clients[k]; + + // Even if it's not shown, make sure that frames are delivered. + multi_compositor_deliver_any_frames(msc->clients[k], display_time_ns); + } + + // Sort the stack array + qsort(array, count, sizeof(struct multi_compositor *), overlay_sort_func); + + for (size_t k = 0; k < count; k++) { + struct multi_compositor *mc = array[k]; + + if (mc == NULL) { + continue; + } + + // None of the data in this slot is valid, don't check access it. + if (!mc->delivered.active) { + continue; + } + + uint64_t frame_time_ns = mc->delivered.display_time_ns; + if (!time_is_within_half_ms(frame_time_ns, display_time_ns)) { + log_frame_time_diff(frame_time_ns, display_time_ns); + } + + for (size_t i = 0; i < mc->delivered.num_layers; i++) { + struct multi_layer_entry *layer = &mc->delivered.layers[i]; + + switch (layer->data.type) { + case XRT_LAYER_STEREO_PROJECTION: do_projection_layer(xc, mc, layer, i); break; + case XRT_LAYER_STEREO_PROJECTION_DEPTH: do_projection_layer_depth(xc, mc, layer, i); break; + case XRT_LAYER_QUAD: do_quad_layer(xc, mc, layer, i); break; + case XRT_LAYER_CUBE: do_cube_layer(xc, mc, layer, i); break; + case XRT_LAYER_CYLINDER: do_cylinder_layer(xc, mc, layer, i); break; + case XRT_LAYER_EQUIRECT1: do_equirect1_layer(xc, mc, layer, i); break; + case XRT_LAYER_EQUIRECT2: do_equirect2_layer(xc, mc, layer, i); break; + default: U_LOG_E("Unhandled layer type '%i'!", layer->data.type); break; + } + } + } +} + +static void +broadcast_timings(struct multi_system_compositor *msc, + uint64_t predicted_display_time_ns, + uint64_t predicted_display_period_ns, + uint64_t diff_ns) +{ + COMP_TRACE_MARKER(); + + os_mutex_lock(&msc->list_and_timing_lock); + + for (size_t i = 0; i < ARRAY_SIZE(msc->clients); i++) { + struct multi_compositor *mc = msc->clients[i]; + if (mc == NULL) { + continue; + } + + u_rt_info( // + mc->urt, // + predicted_display_time_ns, // + predicted_display_period_ns, // + diff_ns); // + } + + msc->last_timings.predicted_display_time_ns = predicted_display_time_ns; + msc->last_timings.predicted_display_period_ns = predicted_display_period_ns; + msc->last_timings.diff_ns = diff_ns; + + os_mutex_unlock(&msc->list_and_timing_lock); +} + +static void +wait_frame(struct xrt_compositor *xc, + int64_t *out_frame_id, + uint64_t *out_wake_time_ns, + uint64_t *out_predicted_gpu_time_ns, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns) +{ + COMP_TRACE_MARKER(); + + int64_t frame_id = -1; + uint64_t wake_up_time_ns = 0; + + xrt_comp_predict_frame( // + xc, // + &frame_id, // + &wake_up_time_ns, // + out_predicted_gpu_time_ns, // + out_predicted_display_time_ns, // + out_predicted_display_period_ns); // + + uint64_t now_ns = os_monotonic_get_ns(); + if (now_ns < wake_up_time_ns) { + os_nanosleep(wake_up_time_ns - now_ns); + } + + now_ns = os_monotonic_get_ns(); + + xrt_comp_mark_frame(xc, frame_id, XRT_COMPOSITOR_FRAME_POINT_WOKE, now_ns); + + *out_frame_id = frame_id; + *out_wake_time_ns = wake_up_time_ns; +} + +static int +multi_main_loop(struct multi_system_compositor *msc) +{ + COMP_TRACE_MARKER(); + + struct xrt_compositor *xc = &msc->xcn->base; + + //! @todo Don't make this a hack. + enum xrt_view_type view_type = XRT_VIEW_TYPE_STEREO; + + xrt_comp_begin_session(xc, view_type); + + os_thread_helper_lock(&msc->oth); + while (os_thread_helper_is_running_locked(&msc->oth)) { + os_thread_helper_unlock(&msc->oth); + + int64_t frame_id; + uint64_t wake_time_ns = 0; + uint64_t predicted_gpu_time_ns = 0; + uint64_t predicted_display_time_ns = 0; + uint64_t predicted_display_period_ns = 0; + + wait_frame( // + xc, // + &frame_id, // + &wake_time_ns, // + &predicted_gpu_time_ns, // + &predicted_display_time_ns, // + &predicted_display_period_ns); // + + uint64_t now_ns = os_monotonic_get_ns(); + uint64_t diff_ns = predicted_display_time_ns - now_ns; + + broadcast_timings(msc, predicted_display_time_ns, predicted_display_period_ns, diff_ns); + + xrt_comp_begin_frame(xc, frame_id); + xrt_comp_layer_begin(xc, frame_id, 0, 0); + + // Make sure that the clients doesn't go away while we transfer layers. + os_mutex_lock(&msc->list_and_timing_lock); + transfer_layers_locked(msc, predicted_display_time_ns); + os_mutex_unlock(&msc->list_and_timing_lock); + + xrt_comp_layer_commit(xc, frame_id, XRT_GRAPHICS_SYNC_HANDLE_INVALID); + + // Re-lock the thread for check in while statement. + os_thread_helper_lock(&msc->oth); + } + + xrt_comp_end_session(xc); + + return 0; +} + +static void * +thread_func(void *ptr) +{ + return (void *)(intptr_t)multi_main_loop((struct multi_system_compositor *)ptr); +} + + +/* + * + * System multi compositor functions. + * + */ + +static xrt_result_t +system_compositor_set_state(struct xrt_system_compositor *xsc, struct xrt_compositor *xc, bool visible, bool focused) +{ + struct multi_system_compositor *msc = multi_system_compositor(xsc); + struct multi_compositor *mc = multi_compositor(xc); + (void)msc; + + //! @todo Locking? + if (mc->state.sent.visible != visible || mc->state.sent.focused != focused) { + mc->state.sent.visible = visible; + mc->state.sent.focused = focused; + + union xrt_compositor_event xce = {0}; + xce.type = XRT_COMPOSITOR_EVENT_STATE_CHANGE; + xce.state.visible = visible; + xce.state.focused = focused; + + multi_compositor_push_event(mc, &xce); + } + + return XRT_SUCCESS; +} + +static xrt_result_t +system_compositor_set_z_order(struct xrt_system_compositor *xsc, struct xrt_compositor *xc, int64_t z_order) +{ + struct multi_system_compositor *msc = multi_system_compositor(xsc); + struct multi_compositor *mc = multi_compositor(xc); + (void)msc; + + //! @todo Locking? + mc->state.z_order = z_order; + + return XRT_SUCCESS; +} + +static xrt_result_t +system_compositor_set_main_app_visibility(struct xrt_system_compositor *xsc, struct xrt_compositor *xc, bool visible) +{ + struct multi_system_compositor *msc = multi_system_compositor(xsc); + struct multi_compositor *mc = multi_compositor(xc); + (void)msc; + + union xrt_compositor_event xce = {0}; + xce.type = XRT_COMPOSITOR_EVENT_OVERLAY_CHANGE; + xce.overlay.visible = visible; + + multi_compositor_push_event(mc, &xce); + + return XRT_SUCCESS; +} + + +/* + * + * System compositor functions. + * + */ + +static xrt_result_t +system_compositor_create_native_compositor(struct xrt_system_compositor *xsc, + const struct xrt_session_info *xsi, + struct xrt_compositor_native **out_xcn) +{ + struct multi_system_compositor *msc = multi_system_compositor(xsc); + + return multi_compositor_create(msc, xsi, out_xcn); +} + +static void +system_compositor_destroy(struct xrt_system_compositor *xsc) +{ + struct multi_system_compositor *msc = multi_system_compositor(xsc); + + // Stop the render thread first. + os_thread_helper_stop(&msc->oth); + + xrt_comp_native_destroy(&msc->xcn); + + os_mutex_destroy(&msc->list_and_timing_lock); + + free(msc); +} + + +/* + * + * 'Exported' functions. + * + */ + +xrt_result_t +comp_multi_create_system_compositor(struct xrt_compositor_native *xcn, + const struct xrt_system_compositor_info *xsci, + struct xrt_system_compositor **out_xsysc) +{ + struct multi_system_compositor *msc = U_TYPED_CALLOC(struct multi_system_compositor); + msc->base.create_native_compositor = system_compositor_create_native_compositor; + msc->base.destroy = system_compositor_destroy; + msc->xmcc.set_state = system_compositor_set_state; + msc->xmcc.set_z_order = system_compositor_set_z_order; + msc->xmcc.set_main_app_visibility = system_compositor_set_main_app_visibility; + msc->base.xmcc = &msc->xmcc; + msc->base.info = *xsci; + msc->xcn = xcn; + + os_mutex_init(&msc->list_and_timing_lock); + + //! @todo Make the clients not go from IDLE to READY before we have completed a first frame. + // Make sure there is at least some sort of valid frame data here. + msc->last_timings.predicted_display_time_ns = os_monotonic_get_ns(); // As good as any time. + msc->last_timings.predicted_display_period_ns = U_TIME_1MS_IN_NS * 16; // Just a wild guess. + msc->last_timings.diff_ns = U_TIME_1MS_IN_NS * 5; // Make sure it's not zero at least. + + int ret = os_thread_helper_init(&msc->oth); + if (ret < 0) { + return XRT_ERROR_THREADING_INIT_FAILURE; + } + + os_thread_helper_start(&msc->oth, thread_func, msc); + + *out_xsysc = &msc->base; + + return XRT_SUCCESS; +} diff --git a/src/xrt/compositor/render/comp_compute.c b/src/xrt/compositor/render/comp_compute.c new file mode 100644 index 000000000..7438ab7cb --- /dev/null +++ b/src/xrt/compositor/render/comp_compute.c @@ -0,0 +1,903 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief The compositor compute based rendering code. + * @author Jakob Bornecrantz + * @ingroup comp_main + */ + +#include "math/m_api.h" +#include "math/m_matrix_4x4_f64.h" + +#include "main/comp_compositor.h" +#include "render/comp_render.h" + +#include + + +/* + * + * Defines + * + */ + +#define C(c) \ + do { \ + VkResult ret = c; \ + if (ret != VK_SUCCESS) { \ + return false; \ + } \ + } while (false) + +#define D(TYPE, thing) \ + if (thing != VK_NULL_HANDLE) { \ + vk->vkDestroy##TYPE(vk, vk->device, thing, NULL); \ + thing = VK_NULL_HANDLE; \ + } + +#define DD(pool, thing) \ + if (thing != VK_NULL_HANDLE) { \ + free_descriptor_set(vk, pool, thing); \ + thing = VK_NULL_HANDLE; \ + } + + +/* + * + * Helper functions. + * + */ + +/* + * For dispatching compute to the view, calculate the number of groups. + */ +static void +calc_dispatch_dims(const struct comp_viewport_data views[2], uint32_t *out_w, uint32_t *out_h) +{ +#define IMAX(a, b) ((a) > (b) ? (a) : (b)) + uint32_t w = IMAX(views[0].w, views[1].w); + uint32_t h = IMAX(views[0].h, views[1].h); +#undef IMAX + + // Power of two divide and round up. +#define P2_DIVIDE_ROUND_UP(v, div) ((v + (div - 1)) / div) + w = P2_DIVIDE_ROUND_UP(w, 8); + h = P2_DIVIDE_ROUND_UP(h, 8); +#undef P2_DIVIDE_ROUND_UP + + *out_w = w; + *out_h = h; +} + +/*! + * Create a simplified projection matrix for timewarp. + */ +static void +calc_projection(const struct xrt_fov *fov, struct xrt_matrix_4x4_f64 *result) +{ + const double tan_left = tan(fov->angle_left); + const double tan_right = tan(fov->angle_right); + + const double tan_down = tan(fov->angle_down); + const double tan_up = tan(fov->angle_up); + + const double tan_width = tan_right - tan_left; + const double tan_height = tan_up - tan_down; + + const double near = 0.5; + const double far = 1.5; + + const double a11 = 2 / tan_width; + const double a22 = 2 / tan_height; + + const double a31 = (tan_right + tan_left) / tan_width; + const double a32 = (tan_up + tan_down) / tan_height; + + const float a33 = -far / (far - near); + const float a43 = -(far * near) / (far - near); + + +#if 1 + // We skip a33 & a43 because we don't have depth. + (void)a33; + (void)a43; + + // clang-format off + *result = (struct xrt_matrix_4x4_f64){ + { + a11, 0, 0, 0, + 0, a22, 0, 0, + a31, a32, -1, 0, + 0, 0, 0, 1, + } + }; + // clang-format on +#else + // clang-format off + *result = (struct xrt_matrix_4x4_f64) { + .v = { + a11, 0, 0, 0, + 0, a22, 0, 0, + a31, a32, a33, -1, + 0, 0, a43, 0, + } + }; + // clang-format on +#endif +} + +static void +calc_time_warp_matrix(struct comp_rendering_compute *crc, + const struct xrt_pose *src_pose, + const struct xrt_fov *src_fov, + const struct xrt_pose *new_pose, + struct xrt_matrix_4x4 *matrix) +{ + // Src projection matrix. + struct xrt_matrix_4x4_f64 src_proj; + calc_projection(src_fov, &src_proj); + + // Src rotation matrix. + struct xrt_matrix_4x4_f64 src_rot_inv; + struct xrt_quat src_q = src_pose->orientation; + m_mat4_f64_orientation(&src_q, &src_rot_inv); // This is a model matrix, a inverted view matrix. + + // New rotation matrix. + struct xrt_matrix_4x4_f64 new_rot, new_rot_inv; + struct xrt_quat new_q = new_pose->orientation; + m_mat4_f64_orientation(&new_q, &new_rot_inv); // This is a model matrix, a inverted view matrix. + m_mat4_f64_invert(&new_rot_inv, &new_rot); // Invert to make it a view matrix. + + // Combine both rotation matricies to get difference. + struct xrt_matrix_4x4_f64 delta_rot, delta_rot_inv; + m_mat4_f64_multiply(&new_rot, &src_rot_inv, &delta_rot); + m_mat4_f64_invert(&delta_rot, &delta_rot_inv); + + // Combine the source projection matrix and + struct xrt_matrix_4x4_f64 result; + m_mat4_f64_multiply(&src_proj, &delta_rot_inv, &result); + + // Reset if timewarp is off. + if (crc->c->debug.atw_off) { + result = src_proj; + } + + // Convert from f64 to f32. + for (int i = 0; i < 16; i++) { + matrix->v[i] = result.v[i]; + } +} + + +/* + * + * Vulkan helpers. + * + */ + +static VkResult +create_command_buffer(struct vk_bundle *vk, VkCommandBuffer *out_cmd) +{ + VkResult ret; + + VkCommandBufferAllocateInfo cmd_buffer_info = { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, + .commandPool = vk->cmd_pool, + .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, + .commandBufferCount = 1, + }; + + VkCommandBuffer cmd = VK_NULL_HANDLE; + + os_mutex_lock(&vk->cmd_pool_mutex); + + ret = vk->vkAllocateCommandBuffers( // + vk->device, // + &cmd_buffer_info, // + &cmd); // + + os_mutex_unlock(&vk->cmd_pool_mutex); + + if (ret != VK_SUCCESS) { + VK_ERROR(vk, "vkCreateFramebuffer failed: %s", vk_result_string(ret)); + return ret; + } + + *out_cmd = cmd; + + return VK_SUCCESS; +} + +static void +destroy_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer) +{ + os_mutex_lock(&vk->cmd_pool_mutex); + + vk->vkFreeCommandBuffers( // + vk->device, // device + vk->cmd_pool, // commandPool + 1, // commandBufferCount + &command_buffer); // pCommandBuffers + + os_mutex_unlock(&vk->cmd_pool_mutex); +} + +static VkResult +begin_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer) +{ + VkResult ret; + + VkCommandBufferBeginInfo command_buffer_info = { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + }; + + ret = vk->vkBeginCommandBuffer( // + command_buffer, // + &command_buffer_info); // + if (ret != VK_SUCCESS) { + VK_ERROR(vk, "vkBeginCommandBuffer failed: %s", vk_result_string(ret)); + return ret; + } + + return VK_SUCCESS; +} + +static VkResult +end_command_buffer(struct vk_bundle *vk, VkCommandBuffer command_buffer) +{ + VkResult ret; + + // End the command buffer. + ret = vk->vkEndCommandBuffer( // + command_buffer); // + if (ret != VK_SUCCESS) { + VK_ERROR(vk, "vkEndCommandBuffer failed: %s", vk_result_string(ret)); + return ret; + } + + return VK_SUCCESS; +} + +static VkResult +create_descriptor_set(struct vk_bundle *vk, + VkDescriptorPool descriptor_pool, + VkDescriptorSetLayout descriptor_layout, + VkDescriptorSet *out_descriptor_set) +{ + VkResult ret; + + VkDescriptorSetAllocateInfo alloc_info = { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, + .descriptorPool = descriptor_pool, + .descriptorSetCount = 1, + .pSetLayouts = &descriptor_layout, + }; + + VkDescriptorSet descriptor_set = VK_NULL_HANDLE; + ret = vk->vkAllocateDescriptorSets( // + vk->device, // + &alloc_info, // + &descriptor_set); // + if (ret != VK_SUCCESS) { + VK_DEBUG(vk, "vkAllocateDescriptorSets failed: %s", vk_result_string(ret)); + return ret; + } + + *out_descriptor_set = descriptor_set; + + return VK_SUCCESS; +} + +XRT_MAYBE_UNUSED static void +update_compute_discriptor_set(struct vk_bundle *vk, + uint32_t src_binding, + VkSampler src_samplers[2], + VkImageView src_image_views[2], + uint32_t distortion_binding, + VkSampler distortion_samplers[6], + VkImageView distortion_image_views[6], + uint32_t target_binding, + VkImageView target_image_view, + uint32_t ubo_binding, + VkBuffer ubo_buffer, + VkDeviceSize ubo_size, + VkDescriptorSet descriptor_set) +{ + VkDescriptorImageInfo src_image_info[2] = { + { + .sampler = src_samplers[0], + .imageView = src_image_views[0], + .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + }, + { + .sampler = src_samplers[1], + .imageView = src_image_views[1], + .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + }, + }; + + VkDescriptorImageInfo distortion_image_info[6] = { + { + .sampler = distortion_samplers[0], + .imageView = distortion_image_views[0], + .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + }, + { + .sampler = distortion_samplers[1], + .imageView = distortion_image_views[1], + .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + }, + { + .sampler = distortion_samplers[2], + .imageView = distortion_image_views[2], + .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + }, + { + .sampler = distortion_samplers[3], + .imageView = distortion_image_views[3], + .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + }, + { + .sampler = distortion_samplers[4], + .imageView = distortion_image_views[4], + .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + }, + { + .sampler = distortion_samplers[5], + .imageView = distortion_image_views[5], + .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + }, + }; + + VkDescriptorImageInfo target_image_info = { + .imageView = target_image_view, + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + }; + + VkDescriptorBufferInfo buffer_info = { + .buffer = ubo_buffer, + .offset = 0, + .range = ubo_size, + }; + + VkWriteDescriptorSet write_descriptor_sets[4] = { + { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .dstSet = descriptor_set, + .dstBinding = src_binding, + .descriptorCount = ARRAY_SIZE(src_image_info), + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImageInfo = src_image_info, + }, + { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .dstSet = descriptor_set, + .dstBinding = distortion_binding, + .descriptorCount = ARRAY_SIZE(distortion_image_info), + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImageInfo = distortion_image_info, + }, + { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .dstSet = descriptor_set, + .dstBinding = target_binding, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + .pImageInfo = &target_image_info, + }, + { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .dstSet = descriptor_set, + .dstBinding = ubo_binding, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .pBufferInfo = &buffer_info, + }, + }; + + vk->vkUpdateDescriptorSets( // + vk->device, // + ARRAY_SIZE(write_descriptor_sets), // descriptorWriteCount + write_descriptor_sets, // pDescriptorWrites + 0, // descriptorCopyCount + NULL); // pDescriptorCopies +} + +XRT_MAYBE_UNUSED static void +update_compute_discriptor_set_target(struct vk_bundle *vk, + uint32_t target_binding, + VkImageView target_image_view, + uint32_t ubo_binding, + VkBuffer ubo_buffer, + VkDeviceSize ubo_size, + VkDescriptorSet descriptor_set) +{ + VkDescriptorImageInfo target_image_info = { + .imageView = target_image_view, + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + }; + + VkDescriptorBufferInfo buffer_info = { + .buffer = ubo_buffer, + .offset = 0, + .range = ubo_size, + }; + + VkWriteDescriptorSet write_descriptor_sets[2] = { + { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .dstSet = descriptor_set, + .dstBinding = target_binding, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + .pImageInfo = &target_image_info, + }, + { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .dstSet = descriptor_set, + .dstBinding = ubo_binding, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .pBufferInfo = &buffer_info, + }, + }; + + vk->vkUpdateDescriptorSets( // + vk->device, // + ARRAY_SIZE(write_descriptor_sets), // descriptorWriteCount + write_descriptor_sets, // pDescriptorWrites + 0, // descriptorCopyCount + NULL); // pDescriptorCopies +} + + +/* + * + * 'Exported' functions. + * + */ + +bool +comp_rendering_compute_init(struct comp_compositor *c, struct comp_resources *r, struct comp_rendering_compute *crc) +{ + assert(crc->c == NULL); + assert(crc->r == NULL); + + struct vk_bundle *vk = &c->vk; + crc->c = c; + crc->r = r; + + C(create_command_buffer(vk, &crc->cmd)); + + C(create_descriptor_set( // + vk, // + r->compute.descriptor_pool, // descriptor_pool + r->compute.descriptor_set_layout, // descriptor_set_layout + &crc->clear_descriptor_set)); // descriptor_set + + return true; +} + +bool +comp_rendering_compute_begin(struct comp_rendering_compute *crc) +{ + struct vk_bundle *vk = &crc->c->vk; + + C(begin_command_buffer(vk, crc->cmd)); + + return true; +} + +bool +comp_rendering_compute_end(struct comp_rendering_compute *crc) +{ + struct vk_bundle *vk = &crc->c->vk; + + C(end_command_buffer(vk, crc->cmd)); + + return true; +} + +void +comp_rendering_compute_close(struct comp_rendering_compute *crc) +{ + assert(crc->c != NULL); + assert(crc->r != NULL); + + struct vk_bundle *vk = &crc->c->vk; + + destroy_command_buffer(vk, crc->cmd); + + // Reclaimed by vkResetDescriptorPool. + crc->clear_descriptor_set = VK_NULL_HANDLE; + + vk->vkResetDescriptorPool(vk->device, crc->r->compute.descriptor_pool, 0); + + crc->c = NULL; + crc->r = NULL; +} + +void +comp_rendering_compute_projection_timewarp(struct comp_rendering_compute *crc, + VkSampler src_samplers[2], + VkImageView src_image_views[2], + const struct xrt_normalized_rect src_norm_rects[2], + const struct xrt_pose src_poses[2], + const struct xrt_fov src_fovs[2], + const struct xrt_pose new_poses[2], + VkImage target_image, + VkImageView target_image_view, + const struct comp_viewport_data views[2]) +{ + assert(crc->c != NULL); + assert(crc->r != NULL); + + struct vk_bundle *vk = &crc->c->vk; + struct comp_resources *r = crc->r; + + + /* + * UBO + */ + + struct xrt_matrix_4x4 time_warp_matrix[2]; + calc_time_warp_matrix( // + crc, // + &src_poses[0], // + &src_fovs[0], // + &new_poses[0], // + &time_warp_matrix[0]); // + calc_time_warp_matrix( // + crc, // + &src_poses[1], // + &src_fovs[1], // + &new_poses[1], // + &time_warp_matrix[1]); // + + struct comp_ubo_compute_data *data = (struct comp_ubo_compute_data *)r->compute.ubo.mapped; + data->views[0] = views[0]; + data->views[1] = views[1]; + data->pre_transforms[0] = r->distortion.uv_to_tanangle[0]; + data->pre_transforms[1] = r->distortion.uv_to_tanangle[1]; + data->transforms[0] = time_warp_matrix[0]; + data->transforms[1] = time_warp_matrix[1]; + data->post_transforms[0] = src_norm_rects[0]; + data->post_transforms[1] = src_norm_rects[1]; + + + /* + * Source, target and distortion images. + */ + + VkImageSubresourceRange subresource_range = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }; + + vk_set_image_layout( // + vk, // + crc->cmd, // + target_image, // + 0, // + VK_ACCESS_SHADER_WRITE_BIT, // + VK_IMAGE_LAYOUT_UNDEFINED, // + VK_IMAGE_LAYOUT_GENERAL, // + subresource_range); // + + VkSampler sampler = r->compute.default_sampler; + VkSampler distortion_samplers[6] = { + sampler, sampler, sampler, sampler, sampler, sampler, + }; + + update_compute_discriptor_set( // + vk, // + r->compute.src_binding, // + src_samplers, // + src_image_views, // + r->compute.distortion_binding, // + distortion_samplers, // + r->distortion.image_views, // + r->compute.target_binding, // + target_image_view, // + r->compute.ubo_binding, // + r->compute.ubo.buffer, // + VK_WHOLE_SIZE, // + crc->clear_descriptor_set); // + + vk->vkCmdBindPipeline( // + crc->cmd, // commandBuffer + VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint + r->compute.distortion_timewarp_pipeline); // pipeline + + vk->vkCmdBindDescriptorSets( // + crc->cmd, // commandBuffer + VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint + r->compute.pipeline_layout, // layout + 0, // firstSet + 1, // descriptorSetCount + &crc->clear_descriptor_set, // pDescriptorSets + 0, // dynamicOffsetCount + NULL); // pDynamicOffsets + + + uint32_t w = 0, h = 0; + calc_dispatch_dims(views, &w, &h); + assert(w != 0 && h != 0); + + vk->vkCmdDispatch( // + crc->cmd, // commandBuffer + w, // groupCountX + h, // groupCountY + 2); // groupCountZ + + VkImageMemoryBarrier memoryBarrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = target_image, + .subresourceRange = subresource_range, + }; + + vk->vkCmdPipelineBarrier( // + crc->cmd, // + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // + 0, // + 0, // + NULL, // + 0, // + NULL, // + 1, // + &memoryBarrier); // +} + +void +comp_rendering_compute_projection(struct comp_rendering_compute *crc, + VkSampler src_samplers[2], + VkImageView src_image_views[2], + const struct xrt_normalized_rect src_norm_rects[2], + VkImage target_image, + VkImageView target_image_view, + const struct comp_viewport_data views[2]) +{ + assert(crc->c != NULL); + assert(crc->r != NULL); + + struct vk_bundle *vk = &crc->c->vk; + struct comp_resources *r = crc->r; + + + /* + * UBO + */ + + struct comp_ubo_compute_data *data = (struct comp_ubo_compute_data *)r->compute.ubo.mapped; + data->views[0] = views[0]; + data->views[1] = views[1]; + data->post_transforms[0] = src_norm_rects[0]; + data->post_transforms[1] = src_norm_rects[1]; + + + /* + * Source, target and distortion images. + */ + + VkImageSubresourceRange subresource_range = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }; + + vk_set_image_layout( // + vk, // + crc->cmd, // + target_image, // + 0, // + VK_ACCESS_SHADER_WRITE_BIT, // + VK_IMAGE_LAYOUT_UNDEFINED, // + VK_IMAGE_LAYOUT_GENERAL, // + subresource_range); // + + VkSampler sampler = r->compute.default_sampler; + VkSampler distortion_samplers[6] = { + sampler, sampler, sampler, sampler, sampler, sampler, + }; + + update_compute_discriptor_set( // + vk, // + r->compute.src_binding, // + src_samplers, // + src_image_views, // + r->compute.distortion_binding, // + distortion_samplers, // + r->distortion.image_views, // + r->compute.target_binding, // + target_image_view, // + r->compute.ubo_binding, // + r->compute.ubo.buffer, // + VK_WHOLE_SIZE, // + crc->clear_descriptor_set); // + + vk->vkCmdBindPipeline( // + crc->cmd, // commandBuffer + VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint + r->compute.distortion_pipeline); // pipeline + + vk->vkCmdBindDescriptorSets( // + crc->cmd, // commandBuffer + VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint + r->compute.pipeline_layout, // layout + 0, // firstSet + 1, // descriptorSetCount + &crc->clear_descriptor_set, // pDescriptorSets + 0, // dynamicOffsetCount + NULL); // pDynamicOffsets + + + uint32_t w = 0, h = 0; + calc_dispatch_dims(views, &w, &h); + assert(w != 0 && h != 0); + + vk->vkCmdDispatch( // + crc->cmd, // commandBuffer + w, // groupCountX + h, // groupCountY + 2); // groupCountZ + + VkImageMemoryBarrier memoryBarrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = target_image, + .subresourceRange = subresource_range, + }; + + vk->vkCmdPipelineBarrier( // + crc->cmd, // + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // + 0, // + 0, // + NULL, // + 0, // + NULL, // + 1, // + &memoryBarrier); // +} + +void +comp_rendering_compute_clear(struct comp_rendering_compute *crc, // + VkImage target_image, // + VkImageView target_image_view, // + const struct comp_viewport_data views[2]) // +{ + assert(crc->c != NULL); + assert(crc->r != NULL); + + struct vk_bundle *vk = &crc->c->vk; + struct comp_resources *r = crc->r; + + + /* + * UBO + */ + + // Calculate transforms. + struct xrt_matrix_4x4 transforms[2]; + for (uint32_t i = 0; i < 2; i++) { + math_matrix_4x4_identity(&transforms[i]); + } + + struct comp_ubo_compute_data *data = (struct comp_ubo_compute_data *)r->compute.ubo.mapped; + data->views[0] = views[0]; + data->views[1] = views[1]; + + + /* + * Source, target and distortion images. + */ + + VkImageSubresourceRange subresource_range = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }; + + vk_set_image_layout( // + vk, // + crc->cmd, // + target_image, // + 0, // + VK_ACCESS_SHADER_WRITE_BIT, // + VK_IMAGE_LAYOUT_UNDEFINED, // + VK_IMAGE_LAYOUT_GENERAL, // + subresource_range); // + + VkSampler sampler = r->compute.default_sampler; + VkSampler src_samplers[2] = {sampler, sampler}; + VkImageView src_image_views[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + VkSampler distortion_samplers[6] = {sampler, sampler, sampler, sampler, sampler, sampler}; + VkImageView distortion_image_views[6] = {VK_NULL_HANDLE, VK_NULL_HANDLE, VK_NULL_HANDLE, + VK_NULL_HANDLE, VK_NULL_HANDLE, VK_NULL_HANDLE}; + + update_compute_discriptor_set( // + vk, // + r->compute.src_binding, // + src_samplers, // + src_image_views, // + r->compute.distortion_binding, // + distortion_samplers, // + distortion_image_views, // + r->compute.target_binding, // + target_image_view, // + r->compute.ubo_binding, // + r->compute.ubo.buffer, // + VK_WHOLE_SIZE, // + crc->clear_descriptor_set); // + + vk->vkCmdBindPipeline( // + crc->cmd, // commandBuffer + VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint + r->compute.clear_pipeline); // pipeline + + vk->vkCmdBindDescriptorSets( // + crc->cmd, // commandBuffer + VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint + r->compute.pipeline_layout, // layout + 0, // firstSet + 1, // descriptorSetCount + &crc->clear_descriptor_set, // pDescriptorSets + 0, // dynamicOffsetCount + NULL); // pDynamicOffsets + + + uint32_t w = 0, h = 0; + calc_dispatch_dims(views, &w, &h); + assert(w != 0 && h != 0); + + vk->vkCmdDispatch( // + crc->cmd, // commandBuffer + w, // groupCountX + h, // groupCountY + 2); // groupCountZ + + VkImageMemoryBarrier memoryBarrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = target_image, + .subresourceRange = subresource_range, + }; + + vk->vkCmdPipelineBarrier( // + crc->cmd, // + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // + 0, // + 0, // + NULL, // + 0, // + NULL, // + 1, // + &memoryBarrier); // +} diff --git a/src/xrt/compositor/render/comp_render.h b/src/xrt/compositor/render/comp_render.h index 4c470d3cf..7f7c2e3cb 100644 --- a/src/xrt/compositor/render/comp_render.h +++ b/src/xrt/compositor/render/comp_render.h @@ -25,10 +25,23 @@ struct comp_compositor; struct comp_swapchain_image; /*! - * @ingroup comp_main + * @addtogroup comp_main * @{ */ +/* + * + * Defines + * + */ + +//! How large in pixels the distortion image is. +#define COMP_DISTORTION_IMAGE_DIMENSIONS (128) + +//! How many distortion images we have, one for each channel (3 rgb) and per view, total 6. +#define COMP_DISTORTION_NUM_IMAGES (6) + + /* * * Buffer @@ -69,7 +82,7 @@ comp_buffer_init(struct vk_bundle *vk, VkDeviceSize size); /*! - * Frees all resources that this buffer has, doesn't not free the buffer itself. + * Frees all resources that this buffer has, but does not free the buffer itself. */ void comp_buffer_close(struct vk_bundle *vk, struct comp_buffer *buffer); @@ -148,6 +161,60 @@ struct comp_resources uint32_t offset_indices[2]; uint32_t total_num_indices; } mesh; + + struct + { + //! Descriptor pool for compute work. + VkDescriptorPool descriptor_pool; + + //! The source projection view binding point. + uint32_t src_binding; + + //! Image storing the distortion. + uint32_t distortion_binding; + + //! Writing the image out too. + uint32_t target_binding; + + //! Uniform data binding. + uint32_t ubo_binding; + + //! Dummy sampler for null images. + VkSampler default_sampler; + + //! Descriptor set layout for compute distortion. + VkDescriptorSetLayout descriptor_set_layout; + + //! Pipeline layout used for compute distortion. + VkPipelineLayout pipeline_layout; + + //! Doesn't depend on target so is static. + VkPipeline clear_pipeline; + + //! Doesn't depend on target so is static. + VkPipeline distortion_pipeline; + + //! Doesn't depend on target so is static. + VkPipeline distortion_timewarp_pipeline; + + //! Target info. + struct comp_buffer ubo; + } compute; + + struct + { + //! Transform to go from UV to tangle angles. + struct xrt_normalized_rect uv_to_tanangle[2]; + + //! Backing memory to distortion images. + VkDeviceMemory device_memories[COMP_DISTORTION_NUM_IMAGES]; + + //! Distortion images. + VkImage images[COMP_DISTORTION_NUM_IMAGES]; + + //! The views into the distortion images. + VkImageView image_views[COMP_DISTORTION_NUM_IMAGES]; + } distortion; }; /*! @@ -276,8 +343,7 @@ struct comp_viewport_data */ struct comp_mesh_ubo_data { - struct xrt_matrix_2x2 rot; - int flip_y; + struct xrt_matrix_2x2 vertex_rot; }; /*! @@ -329,6 +395,129 @@ comp_draw_distortion(struct comp_rendering *rr, VkImageView image_view, struct comp_mesh_ubo_data *data); + +/* + * + * Compute distortion. + * + */ + +/*! + * A compute rendering is used to create command buffers needed to do one frame + * of compositor rendering using compute shaders, it holds onto resources used + * by the command buffer. + */ +struct comp_rendering_compute +{ + struct comp_compositor *c; + struct comp_resources *r; + + //! Command buffer where all commands are recorded. + VkCommandBuffer cmd; + + //! Clear descriptor set. + VkDescriptorSet clear_descriptor_set; + +#if 0 + struct + { + //! The data for this target. + struct comp_target_data data; + + //! Image view we are targeting, not owned by the rendering. + VkImageView image_view; + } targets[2]; + + //! Number of different targets, number of views are always two. + uint32_t num_targets; +#endif + + struct + { + int temp; + } view; + + //! The current view we are "rendering" to. + uint32_t current_view; +}; + +struct comp_rendering_compute_data +{ + struct + { + VkImageView source; + + VkImageView distortion; + + struct + { + uint32_t x; + uint32_t y; + uint32_t width; + uint32_t height; + } dst; + } views[2]; + + VkImageView target; +}; + +/*! + * UBO data that is sent to the compute distortion shaders. + */ +struct comp_ubo_compute_data +{ + struct comp_viewport_data views[2]; + struct xrt_normalized_rect pre_transforms[2]; + struct xrt_normalized_rect post_transforms[2]; + struct xrt_matrix_4x4 transforms[2]; +}; + +/*! + * Init struct and create resources needed for compute rendering. + */ +bool +comp_rendering_compute_init(struct comp_compositor *c, struct comp_resources *r, struct comp_rendering_compute *crc); + +/*! + * Frees all resources held by the compute rendering, does not free the struct itself. + */ +void +comp_rendering_compute_close(struct comp_rendering_compute *crc); + +bool +comp_rendering_compute_begin(struct comp_rendering_compute *crc); + +void +comp_rendering_compute_projection_timewarp(struct comp_rendering_compute *crc, + VkSampler src_samplers[2], + VkImageView src_image_views[2], + const struct xrt_normalized_rect src_rects[2], + const struct xrt_pose src_poses[2], + const struct xrt_fov src_fovs[2], + const struct xrt_pose new_poses[2], + VkImage target_image, + VkImageView target_image_view, + const struct comp_viewport_data views[2]); + +void +comp_rendering_compute_projection(struct comp_rendering_compute *crc, // + VkSampler src_samplers[2], // + VkImageView src_image_views[2], // + const struct xrt_normalized_rect src_rects[2], // + VkImage target_image, // + VkImageView target_image_view, // + const struct comp_viewport_data views[2]); // + +void +comp_rendering_compute_clear(struct comp_rendering_compute *crc, // + VkImage target_image, // + VkImageView target_image_view, // + const struct comp_viewport_data views[2]); // + +bool +comp_rendering_compute_end(struct comp_rendering_compute *crc); + + /*! * @} */ diff --git a/src/xrt/compositor/render/comp_resources.c b/src/xrt/compositor/render/comp_resources.c index 44d05cf22..73e55051a 100644 --- a/src/xrt/compositor/render/comp_resources.c +++ b/src/xrt/compositor/render/comp_resources.c @@ -11,6 +11,7 @@ #include "main/comp_compositor.h" #include "render/comp_render.h" +#include "math/m_vec2.h" #include @@ -29,6 +30,13 @@ thing = VK_NULL_HANDLE; \ } +#define DF(TYPE, thing) \ + if (thing != VK_NULL_HANDLE) { \ + vk->vkFree##TYPE(vk->device, thing, NULL); \ + thing = VK_NULL_HANDLE; \ + } + + static VkResult create_pipeline_cache(struct vk_bundle *vk, VkPipelineCache *out_pipeline_cache) { @@ -85,28 +93,48 @@ static VkResult create_descriptor_pool(struct vk_bundle *vk, uint32_t num_uniform_per_desc, uint32_t num_sampler_per_desc, + uint32_t num_storage_per_desc, uint32_t num_descs, + bool freeable, VkDescriptorPool *out_descriptor_pool) { VkResult ret; - VkDescriptorPoolSize pool_sizes[2] = { - { - .type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, - .descriptorCount = num_uniform_per_desc * num_descs, - }, - { - .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .descriptorCount = num_sampler_per_desc * num_descs, - }, - }; + uint32_t count = 0; + VkDescriptorPoolSize pool_sizes[3] = {0}; + + if (num_uniform_per_desc > 0) { + pool_sizes[count].type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; + pool_sizes[count].descriptorCount = num_uniform_per_desc * num_descs; + count++; + } + + if (num_sampler_per_desc > 0) { + pool_sizes[count].type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + pool_sizes[count].descriptorCount = num_sampler_per_desc * num_descs; + count++; + } + + if (num_storage_per_desc > 0) { + pool_sizes[count].type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + pool_sizes[count].descriptorCount = num_storage_per_desc * num_descs; + count++; + } + + assert(count > 0 && count <= ARRAY_SIZE(pool_sizes)); + + VkDescriptorPoolCreateFlags flags = 0; + + if (freeable) { + flags |= VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT; + } VkDescriptorPoolCreateInfo descriptor_pool_info = { .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, - .flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT, + .flags = flags, .maxSets = num_descs, - .poolSizeCount = ARRAY_SIZE(pool_sizes), + .poolSizeCount = count, .pPoolSizes = pool_sizes, }; @@ -235,6 +263,341 @@ init_mesh_vertex_buffers(struct vk_bundle *vk, } +/* + * + * Compute + * + */ + +static VkResult +create_compute_descriptor_set_layout(struct vk_bundle *vk, + uint32_t src_binding, + uint32_t distortion_binding, + uint32_t target_binding, + uint32_t ubo_binding, + VkDescriptorSetLayout *out_descriptor_set_layout) +{ + VkResult ret; + + VkDescriptorSetLayoutBinding set_layout_bindings[4] = { + { + .binding = src_binding, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .descriptorCount = 2, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + }, + { + .binding = distortion_binding, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .descriptorCount = 6, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + }, + { + .binding = target_binding, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + }, + { + .binding = ubo_binding, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + }, + }; + + VkDescriptorSetLayoutCreateInfo set_layout_info = { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .bindingCount = ARRAY_SIZE(set_layout_bindings), + .pBindings = set_layout_bindings, + }; + + VkDescriptorSetLayout descriptor_set_layout = VK_NULL_HANDLE; + ret = vk->vkCreateDescriptorSetLayout( // + vk->device, // + &set_layout_info, // + NULL, // + &descriptor_set_layout); // + if (ret != VK_SUCCESS) { + VK_ERROR(vk, "vkCreateDescriptorSetLayout failed: %s", vk_result_string(ret)); + return ret; + } + + *out_descriptor_set_layout = descriptor_set_layout; + + return VK_SUCCESS; +} + +static VkResult +create_compute_pipeline(struct vk_bundle *vk, + VkPipelineCache pipeline_cache, + VkShaderModule shader, + VkPipelineLayout pipeline_layout, + VkPipeline *out_compute_pipeline) +{ + VkResult ret; + + VkPipelineShaderStageCreateInfo shader_stage_info = { + .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .pNext = NULL, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, + .module = shader, + .pName = "main", + }; + + VkComputePipelineCreateInfo pipeline_info = { + .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .pNext = NULL, + .flags = 0, + .stage = shader_stage_info, + .layout = pipeline_layout, + }; + + VkPipeline pipeline = VK_NULL_HANDLE; + ret = vk->vkCreateComputePipelines( // + vk->device, // + pipeline_cache, // + 1, // + &pipeline_info, // + NULL, // + &pipeline); // + if (ret != VK_SUCCESS) { + VK_DEBUG(vk, "vkCreateComputePipelines failed: %s", vk_result_string(ret)); + return ret; + } + + *out_compute_pipeline = pipeline; + + return VK_SUCCESS; +} + +static VkResult +create_distortion_image_and_view(struct vk_bundle *vk, + VkExtent2D extent, + VkDeviceMemory *out_device_memory, + VkImage *out_image, + VkImageView *out_image_view) +{ + VkFormat format = VK_FORMAT_R32G32_SFLOAT; + VkImage image = VK_NULL_HANDLE; + VkDeviceMemory device_memory = VK_NULL_HANDLE; + VkImageView image_view = VK_NULL_HANDLE; + + C(vk_create_image_simple( // + vk, // vk_bundle + extent, // extent + format, // format + VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, // usage + &device_memory, // out_device_memory + &image)); // out_image + + VkImageSubresourceRange subresource_range = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }; + + C(vk_create_view( // + vk, // vk_bundle + image, // image + format, // format + subresource_range, // subresource_range + &image_view)); // out_image_view + + *out_device_memory = device_memory; + *out_image = image; + *out_image_view = image_view; + + return VK_SUCCESS; +} + +static VkResult +queue_upload_for_first_level_and_layer( + struct vk_bundle *vk, VkCommandBuffer cmd, VkBuffer src, VkImage dst, VkExtent2D extent) +{ + VkImageSubresourceRange subresource_range = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }; + + C(vk_set_image_layout( // + vk, // + cmd, // + dst, // + 0, // + VK_ACCESS_TRANSFER_WRITE_BIT, // + VK_IMAGE_LAYOUT_UNDEFINED, // + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, // + subresource_range)); // + + VkImageSubresourceLayers subresource_layers = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = 1, + }; + + VkBufferImageCopy region = { + .bufferOffset = 0, + .bufferRowLength = 0, + .bufferImageHeight = 0, + .imageSubresource = subresource_layers, + .imageOffset = {0, 0, 0}, + .imageExtent = {extent.width, extent.height, 1}, + }; + + vk->vkCmdCopyBufferToImage( // + cmd, // commandBuffer + src, // srcBuffer + dst, // dstImage + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, // dstImageLayout + 1, // regionCount + ®ion); // pRegions + + C(vk_set_image_layout( // + vk, // + cmd, // + dst, // + 0, // + VK_ACCESS_SHADER_READ_BIT, // + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, // + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, // + subresource_range)); // + + return VK_SUCCESS; +} + +static VkResult +create_and_queue_upload(struct vk_bundle *vk, + VkCommandBuffer cmd, + VkBuffer src_buffer, + VkDeviceMemory *out_image_device_memory, + VkImage *out_image, + VkImageView *out_image_view) +{ + VkExtent2D extent = {COMP_DISTORTION_IMAGE_DIMENSIONS, COMP_DISTORTION_IMAGE_DIMENSIONS}; + + VkDeviceMemory device_memory = VK_NULL_HANDLE; + VkImage image = VK_NULL_HANDLE; + VkImageView image_view = VK_NULL_HANDLE; + + C(create_distortion_image_and_view( // + vk, // vk_bundle + extent, // extent + &device_memory, // out_device_memory + &image, // out_image + &image_view)); // out_image_view + + C(queue_upload_for_first_level_and_layer( // + vk, // vk_bundle + cmd, // cmd + src_buffer, // src + image, // dst + extent)); // extent + + *out_image_device_memory = device_memory; + *out_image = image; + *out_image_view = image_view; + + return VK_SUCCESS; +} + +/*! + * Helper struct to make code easier to read. + */ +struct texture +{ + struct xrt_vec2 pixels[COMP_DISTORTION_IMAGE_DIMENSIONS][COMP_DISTORTION_IMAGE_DIMENSIONS]; +}; + +struct tan_angles_transforms +{ + struct xrt_vec2 offset; + struct xrt_vec2 scale; +}; + +static void +calc_uv_to_tanangle(struct xrt_device *xdev, uint32_t view, struct xrt_normalized_rect *out_rect) +{ + const struct xrt_fov fov = xdev->hmd->views[view].fov; + const double tan_left = tan(fov.angle_left); + const double tan_right = tan(fov.angle_right); + + const double tan_down = tan(fov.angle_down); + const double tan_up = tan(fov.angle_up); + + const double tan_width = tan_right - tan_left; + const double tan_height = tan_up - tan_down; + + const double tan_offset_x = (tan_right + tan_left) - tan_width / 2; + const double tan_offset_y = (tan_up + tan_down) - tan_height / 2; + + struct xrt_normalized_rect transform = { + .x = tan_offset_x, + .y = tan_offset_y, + .w = tan_width, + .h = tan_height, + }; + + *out_rect = transform; +} + +static XRT_MAYBE_UNUSED VkResult +create_and_file_in_distortion_buffer_for_view(struct vk_bundle *vk, + struct xrt_device *xdev, + struct comp_buffer *r_buffer, + struct comp_buffer *g_buffer, + struct comp_buffer *b_buffer, + uint32_t view) +{ + VkBufferUsageFlags usage_flags = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT; + + + VkDeviceSize size = sizeof(struct texture); + + C(comp_buffer_init(vk, r_buffer, usage_flags, properties, size)); + C(comp_buffer_init(vk, g_buffer, usage_flags, properties, size)); + C(comp_buffer_init(vk, b_buffer, usage_flags, properties, size)); + + C(comp_buffer_map(vk, r_buffer)); + C(comp_buffer_map(vk, g_buffer)); + C(comp_buffer_map(vk, b_buffer)); + + struct texture *r = r_buffer->mapped; + struct texture *g = g_buffer->mapped; + struct texture *b = b_buffer->mapped; + + for (int row = 0; row < COMP_DISTORTION_IMAGE_DIMENSIONS; row++) { + // This goes from 0 to 1.0 inclusive. + float v = (double)row / (double)COMP_DISTORTION_IMAGE_DIMENSIONS; + + for (int col = 0; col < COMP_DISTORTION_IMAGE_DIMENSIONS; col++) { + // This goes from 0 to 1.0 inclusive. + float u = (double)col / (double)COMP_DISTORTION_IMAGE_DIMENSIONS; + + struct xrt_uv_triplet result; + xrt_device_compute_distortion(xdev, view, u, v, &result); + + + r->pixels[row][col] = result.r; + g->pixels[row][col] = result.g; + b->pixels[row][col] = result.b; + } + } + + comp_buffer_unmap(vk, r_buffer); + comp_buffer_unmap(vk, g_buffer); + comp_buffer_unmap(vk, b_buffer); + + return VK_SUCCESS; +} + /* * * 'Exported' renderer functions. @@ -262,6 +625,11 @@ comp_resources_init(struct comp_compositor *c, struct comp_resources *r) r->mesh.offset_indices[0] = parts->distortion.mesh.offset_indices[0]; r->mesh.offset_indices[1] = parts->distortion.mesh.offset_indices[1]; + r->compute.src_binding = 0; + r->compute.distortion_binding = 1; + r->compute.target_binding = 2; + r->compute.ubo_binding = 3; + /* * Shared @@ -277,7 +645,9 @@ comp_resources_init(struct comp_compositor *c, struct comp_resources *r) C(create_descriptor_pool(vk, // vk_bundle 1, // num_uniform_per_desc 1, // num_sampler_per_desc + 0, // num_storage_per_desc 16 * 2, // num_descs + true, // freeable &r->mesh_descriptor_pool)); // out_descriptor_pool C(create_mesh_descriptor_set_layout(vk, // vk_bundle @@ -301,6 +671,108 @@ comp_resources_init(struct comp_compositor *c, struct comp_resources *r) } + /* + * Compute static. + */ + + C(vk_create_sampler( // + vk, // vk_bundle + VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, // clamp_mode + &r->compute.default_sampler)); // out_sampler + + C(create_descriptor_pool( // + vk, // vk_bundle + 1, // num_uniform_per_desc + 8, // num_sampler_per_desc + 1, // num_storage_per_desc + 1, // num_descs + false, // freeable + &r->compute.descriptor_pool)); // out_descriptor_pool + + C(create_compute_descriptor_set_layout( // + vk, // vk_bundle + r->compute.src_binding, // src_binding, + r->compute.distortion_binding, // distortion_binding, + r->compute.target_binding, // target_binding, + r->compute.ubo_binding, // ubo_binding, + &r->compute.descriptor_set_layout)); // out_descriptor_set_layout + + C(create_pipeline_layout( // + vk, // vk_bundle + r->compute.descriptor_set_layout, // descriptor_set_layout + &r->compute.pipeline_layout)); // out_pipeline_layout + + C(create_compute_pipeline( // + vk, // vk_bundle + r->pipeline_cache, // pipeline_cache + c->shaders.clear_comp, // shader + r->compute.pipeline_layout, // pipeline_layout + &r->compute.clear_pipeline)); // out_compute_pipeline + + C(create_compute_pipeline( // + vk, // vk_bundle + r->pipeline_cache, // pipeline_cache + c->shaders.distortion_comp, // shader + r->compute.pipeline_layout, // pipeline_layout + &r->compute.distortion_pipeline)); // out_compute_pipeline + + C(create_compute_pipeline( // + vk, // vk_bundle + r->pipeline_cache, // pipeline_cache + c->shaders.distortion_timewarp_comp, // shader + r->compute.pipeline_layout, // pipeline_layout + &r->compute.distortion_timewarp_pipeline)); // out_compute_pipeline + + + VkBufferUsageFlags ubo_usage_flags = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT; + VkMemoryPropertyFlags memory_property_flags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT; + size_t ubo_size = sizeof(struct comp_ubo_compute_data); + + C(comp_buffer_init( // + vk, // vk_bundle + &r->compute.ubo, // buffer + ubo_usage_flags, // usage_flags + memory_property_flags, // memory_property_flags + ubo_size)); // size + C(comp_buffer_map( // + vk, // vk_bundle + &r->compute.ubo)); // buffer + + + struct comp_buffer buffers[COMP_DISTORTION_NUM_IMAGES]; + + calc_uv_to_tanangle(c->xdev, 0, &r->distortion.uv_to_tanangle[0]); + calc_uv_to_tanangle(c->xdev, 1, &r->distortion.uv_to_tanangle[1]); + + create_and_file_in_distortion_buffer_for_view(vk, c->xdev, &buffers[0], &buffers[2], &buffers[4], 0); + create_and_file_in_distortion_buffer_for_view(vk, c->xdev, &buffers[1], &buffers[3], &buffers[5], 1); + + VkCommandBuffer upload_buffer = VK_NULL_HANDLE; + C(vk_init_cmd_buffer(vk, &upload_buffer)); + + for (uint32_t i = 0; i < COMP_DISTORTION_NUM_IMAGES; i++) { + C(create_and_queue_upload( // + vk, // vk_bundle + upload_buffer, // cmd + buffers[i].buffer, // src_buffer + &r->distortion.device_memories[i], // out_image_device_memory + &r->distortion.images[i], // out_image + &r->distortion.image_views[i])); // out_image_view + } + + C(vk_submit_cmd_buffer(vk, upload_buffer)); + + os_mutex_lock(&vk->queue_mutex); + vk->vkDeviceWaitIdle(vk->device); + os_mutex_unlock(&vk->queue_mutex); + + for (uint32_t i = 0; i < ARRAY_SIZE(buffers); i++) { + comp_buffer_close(vk, &buffers[i]); + } + + /* * Done */ @@ -321,4 +793,22 @@ comp_resources_close(struct comp_compositor *c, struct comp_resources *r) D(DescriptorPool, r->mesh_descriptor_pool); comp_buffer_close(vk, &r->mesh.vbo); comp_buffer_close(vk, &r->mesh.ibo); + + D(DescriptorPool, r->compute.descriptor_pool); + D(DescriptorSetLayout, r->compute.descriptor_set_layout); + D(Pipeline, r->compute.clear_pipeline); + D(Pipeline, r->compute.distortion_pipeline); + D(Pipeline, r->compute.distortion_timewarp_pipeline); + D(PipelineLayout, r->compute.pipeline_layout); + D(Sampler, r->compute.default_sampler); + for (uint32_t i = 0; i < ARRAY_SIZE(r->distortion.image_views); i++) { + D(ImageView, r->distortion.image_views[i]); + } + for (uint32_t i = 0; i < ARRAY_SIZE(r->distortion.images); i++) { + D(Image, r->distortion.images[i]); + } + for (uint32_t i = 0; i < ARRAY_SIZE(r->distortion.images); i++) { + DF(Memory, r->distortion.device_memories[i]); + } + comp_buffer_close(vk, &r->compute.ubo); } diff --git a/src/xrt/compositor/shaders/clear.comp b/src/xrt/compositor/shaders/clear.comp new file mode 100644 index 000000000..4d3c45bcc --- /dev/null +++ b/src/xrt/compositor/shaders/clear.comp @@ -0,0 +1,34 @@ +// Copyright 2021, Collabora Ltd. +// Author: Jakob Bornecrantz +// SPDX-License-Identifier: BSL-1.0 + +#version 460 + +layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in; + +layout(set = 0, binding = 2) uniform writeonly restrict image2D target; +layout(set = 0, binding = 3) uniform restrict Config +{ + ivec4 views[2]; + vec4 pre_transform[2]; + vec4 post_transform[2]; + mat4 transform[2]; +} data; + +void main() +{ + uint ix = gl_GlobalInvocationID.x; + uint iy = gl_GlobalInvocationID.y; + uint iz = gl_GlobalInvocationID.z; + + ivec2 offset = ivec2(data.views[iz].xy); + ivec2 extent = ivec2(data.views[iz].zw); + + if (ix >= extent.x || iy >= extent.y) { + return; + } + + vec4 colour = vec4(vec3(0.2), 1.0); + + imageStore(target, ivec2(offset.x + ix, offset.y + iy), colour); +} diff --git a/src/xrt/compositor/shaders/distortion.comp b/src/xrt/compositor/shaders/distortion.comp new file mode 100644 index 000000000..822552a53 --- /dev/null +++ b/src/xrt/compositor/shaders/distortion.comp @@ -0,0 +1,86 @@ +// Copyright 2021, Collabora Ltd. +// Author: Jakob Bornecrantz +// SPDX-License-Identifier: BSL-1.0 + +#version 460 +#extension GL_GOOGLE_include_directive : require + +#include "srgb.inc.glsl" + +layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in; + +layout(set = 0, binding = 0) uniform sampler2D source[2]; +layout(set = 0, binding = 1) uniform sampler2D distortion[6]; +layout(set = 0, binding = 2) uniform writeonly restrict image2D target; +layout(set = 0, binding = 3, std140) uniform restrict Config +{ + ivec4 views[2]; + vec4 pre_transform[2]; + vec4 post_transform[2]; + mat4 transform[2]; +} ubo; + + +vec2 position_to_uv(ivec2 extent, uint ix, uint iy) +{ + float x = float(ix) / float(extent.x); + float y = float(iy) / float(extent.y); + + vec2 dist_uv = vec2(x, y); + +#define DIM (128.0) +#define STRETCH ((DIM - 1.0) / DIM) +#define OFFSET (1.0 / (DIM * 2.0)) + + dist_uv = (dist_uv * STRETCH) + OFFSET; + + return dist_uv; +} + +vec2 transform_uv(vec2 uv, uint iz) +{ + vec2 values = uv; + + // To deal with OpenGL flip and sub image view. + values.xy = values.xy * ubo.post_transform[iz].zw + ubo.post_transform[iz].xy; + + // Ready to be used. + return values.xy; +} + +void main() +{ + uint ix = gl_GlobalInvocationID.x; + uint iy = gl_GlobalInvocationID.y; + uint iz = gl_GlobalInvocationID.z; + + ivec2 offset = ivec2(ubo.views[iz].xy); + ivec2 extent = ivec2(ubo.views[iz].zw); + + if (ix >= extent.x || iy >= extent.y) { + return; + } + + vec2 dist_uv = position_to_uv(extent, ix, iy); + + vec2 r_uv = texture(distortion[iz + 0], dist_uv).xy; + vec2 g_uv = texture(distortion[iz + 2], dist_uv).xy; + vec2 b_uv = texture(distortion[iz + 4], dist_uv).xy; + + // Do any transformation needed. + r_uv = transform_uv(r_uv, iz); + g_uv = transform_uv(g_uv, iz); + b_uv = transform_uv(b_uv, iz); + + // Sample the source with distorted and chromatic abberation corrected samples. + vec4 colour = vec4( + texture(source[iz], r_uv).r, + texture(source[iz], g_uv).g, + texture(source[iz], b_uv).b, + 1); + + // Do colour correction here since there are no automatic conversion in hardware available. + colour = vec4(from_linear_to_srgb(colour.rgb), 1); + + imageStore(target, ivec2(offset.x + ix, offset.y + iy), colour); +} diff --git a/src/xrt/compositor/shaders/distortion_timewarp.comp b/src/xrt/compositor/shaders/distortion_timewarp.comp new file mode 100644 index 000000000..574f60efd --- /dev/null +++ b/src/xrt/compositor/shaders/distortion_timewarp.comp @@ -0,0 +1,95 @@ +// Copyright 2021, Collabora Ltd. +// Author: Jakob Bornecrantz +// SPDX-License-Identifier: BSL-1.0 + +#version 460 +#extension GL_GOOGLE_include_directive : require + +#include "srgb.inc.glsl" + +layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in; + +layout(set = 0, binding = 0) uniform sampler2D source[2]; +layout(set = 0, binding = 1) uniform sampler2D distortion[6]; +layout(set = 0, binding = 2) uniform writeonly restrict image2D target; +layout(set = 0, binding = 3, std140) uniform restrict Config +{ + ivec4 views[2]; + vec4 pre_transform[2]; + vec4 post_transform[2]; + mat4 transform[2]; +} ubo; + + +vec2 position_to_uv(ivec2 extent, uint ix, uint iy) +{ + float x = float(ix) / float(extent.x); + float y = float(iy) / float(extent.y); + + vec2 dist_uv = vec2(x, y); + +#define DIM (128.0) +#define STRETCH ((DIM - 1.0) / DIM) +#define OFFSET (1.0 / (DIM * 2.0)) + + dist_uv = (dist_uv * STRETCH) + OFFSET; + + return dist_uv; +} + +vec2 transform_uv(vec2 uv, uint iz) +{ + vec4 values = vec4(uv, -1, 1); + + // From uv to tan angle (tanget space). + values.xy = values.xy * ubo.pre_transform[iz].zw + ubo.pre_transform[iz].xy; + values.y = -values.y; // Flip to OpenXR coordinate system. + + // Timewarp. + values = ubo.transform[iz] * values; + values.xy = values.xy * (1.0 / max(values.w, 0.00001)); + + // From [-1, 1] to [0, 1] + values.xy = values.xy * 0.5 + 0.5; + values.y = 1 - values.y; // Flip to UV coordinate system. + + // To deal with OpenGL flip and sub image view. + values.xy = values.xy * ubo.post_transform[iz].zw + ubo.post_transform[iz].xy; + + // Done. + return values.xy; +} + +void main() +{ + uint ix = gl_GlobalInvocationID.x; + uint iy = gl_GlobalInvocationID.y; + uint iz = gl_GlobalInvocationID.z; + + ivec2 offset = ivec2(ubo.views[iz].xy); + ivec2 extent = ivec2(ubo.views[iz].zw); + + if (ix >= extent.x || iy >= extent.y) { + return; + } + + vec2 dist_uv = position_to_uv(extent, ix, iy); + + vec2 r_uv = texture(distortion[iz + 0], dist_uv).xy; + vec2 g_uv = texture(distortion[iz + 2], dist_uv).xy; + vec2 b_uv = texture(distortion[iz + 4], dist_uv).xy; + + r_uv = transform_uv(r_uv, iz); + g_uv = transform_uv(g_uv, iz); + b_uv = transform_uv(b_uv, iz); + + vec4 colour = vec4( + texture(source[iz], r_uv).r, + texture(source[iz], g_uv).g, + texture(source[iz], b_uv).b, + 1); + + colour = vec4(from_linear_to_srgb(colour.rgb), 1); + + imageStore(target, ivec2(offset.x + ix, offset.y + iy), colour); +} diff --git a/src/xrt/compositor/shaders/mesh.vert b/src/xrt/compositor/shaders/mesh.vert index f9d6b55c5..124836c54 100644 --- a/src/xrt/compositor/shaders/mesh.vert +++ b/src/xrt/compositor/shaders/mesh.vert @@ -1,14 +1,14 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 // Author: Lubosz Sarnecki // Author: Pete Black +// Author: Jakob Bornecrantz #version 450 layout (binding = 1, std140) uniform ubo { - vec4 rot; - bool flip_y; + vec4 vertex_rot; } ubo_vp; layout (location = 0) in vec4 in_pos_ruv; @@ -26,8 +26,8 @@ out gl_PerVertex void main() { mat2x2 rot = { - ubo_vp.rot.xy, - ubo_vp.rot.zw, + ubo_vp.vertex_rot.xy, + ubo_vp.vertex_rot.zw, }; vec2 pos = rot * in_pos_ruv.xy; @@ -35,11 +35,5 @@ void main() out_guv = in_guv_buv.xy; out_buv = in_guv_buv.zw; - if (ubo_vp.flip_y) { - out_ruv.y = 1.0 - out_ruv.y; - out_guv.y = 1.0 - out_guv.y; - out_buv.y = 1.0 - out_buv.y; - } - gl_Position = vec4(pos, 0.0f, 1.0f); } diff --git a/src/xrt/compositor/shaders/meson.build b/src/xrt/compositor/shaders/meson.build index 070be4246..776b7db8f 100644 --- a/src/xrt/compositor/shaders/meson.build +++ b/src/xrt/compositor/shaders/meson.build @@ -2,6 +2,9 @@ # SPDX-License-Identifier: BSL-1.0 shader_srcs = [ + 'clear.comp', + 'distortion.comp', + 'distortion_timewarp.comp', 'mesh.frag', 'mesh.vert', 'layer.vert', diff --git a/src/xrt/compositor/shaders/srgb.inc.glsl b/src/xrt/compositor/shaders/srgb.inc.glsl new file mode 100644 index 000000000..af1b90ed1 --- /dev/null +++ b/src/xrt/compositor/shaders/srgb.inc.glsl @@ -0,0 +1,22 @@ +// Copyright 2021, Collabora Ltd. +// Author: Jakob Bornecrantz +// SPDX-License-Identifier: BSL-1.0 + + +float from_linear_to_srgb_channel(float value) +{ + if (value < 0.0031308) { + return 12.92 * value; + } else { + return 1.055 * pow(value, 1.0 / 2.4) - 0.055; + } +} + +vec3 from_linear_to_srgb(vec3 linear_rgb) +{ + return vec3( + from_linear_to_srgb_channel(linear_rgb.r), + from_linear_to_srgb_channel(linear_rgb.g), + from_linear_to_srgb_channel(linear_rgb.b) + ); +} diff --git a/src/xrt/drivers/CMakeLists.txt b/src/xrt/drivers/CMakeLists.txt index 65617a9b8..f239c9f76 100644 --- a/src/xrt/drivers/CMakeLists.txt +++ b/src/xrt/drivers/CMakeLists.txt @@ -5,7 +5,6 @@ set(ENABLED_HEADSET_DRIVERS) set(ENABLED_DRIVERS) - if(XRT_BUILD_DRIVER_ARDUINO) set(ARDUINO_SOURCE_FILES arduino/arduino_device.c @@ -31,6 +30,25 @@ if(XRT_BUILD_DRIVER_DAYDREAM) list(APPEND ENABLED_DRIVERS daydream) endif() +if(XRT_BUILD_DRIVER_DEPTHAI) + set(DEPTHAI_SOURCE_FILES + depthai/depthai_driver.cpp + depthai/depthai_interface.h + ) + + add_library(drv_depthai STATIC ${DEPTHAI_SOURCE_FILES}) + target_link_libraries(drv_depthai PRIVATE + xrt-interfaces + aux_os + ${OpenCV_LIBRARIES} + depthai::opencv + depthai::core + XLink + ) + target_include_directories(drv_depthai PRIVATE ${OpenCV_INCLUDE_DIRS}) + list(APPEND ENABLED_DRIVERS depthai) +endif() + if(XRT_BUILD_DRIVER_DUMMY) set(DUMMY_SOURCE_FILES dummy/dummy_hmd.c @@ -43,6 +61,30 @@ if(XRT_BUILD_DRIVER_DUMMY) list(APPEND ENABLED_HEADSET_DRIVERS dummy) endif() +if(XRT_BUILD_DRIVER_QWERTY) + set(QWERTY_SOURCE_FILES + qwerty/qwerty_device.c + qwerty/qwerty_device.h + qwerty/qwerty_interface.h + qwerty/qwerty_prober.c + qwerty/qwerty_sdl.c + ) + + add_library(drv_qwerty STATIC ${QWERTY_SOURCE_FILES}) + target_link_libraries(drv_qwerty PRIVATE + xrt-interfaces + aux_util + ${SDL2_LIBRARIES} + ) + target_include_directories(drv_qwerty PRIVATE + ${SDL2_INCLUDE_DIRS} + ) + list(APPEND ENABLED_DRIVERS qwerty) + + add_library(drv_qwerty_includes INTERFACE) + target_include_directories(drv_qwerty_includes INTERFACE qwerty) +endif() + if(XRT_BUILD_DRIVER_HDK) set(HDK_SOURCE_FILES hdk/hdk_device.cpp @@ -70,9 +112,9 @@ endif() if(XRT_BUILD_DRIVER_NS) set(NS_SOURCE_FILES - north_star/distortion/utility_northstar.h - north_star/distortion/deformation_northstar.h - north_star/distortion/deformation_northstar.cpp + north_star/distortion_3d/utility_northstar.h + north_star/distortion_3d/deformation_northstar.h + north_star/distortion_3d/deformation_northstar.cpp north_star/ns_hmd.h north_star/ns_hmd.c north_star/ns_interface.h @@ -84,6 +126,15 @@ if(XRT_BUILD_DRIVER_NS) list(APPEND ENABLED_HEADSET_DRIVERS ns) endif() +if(XRT_BUILD_DRIVER_ULV2) + set(ULV2_SOURCE_FILES + ultraleap_v2/ulv2_driver.cpp + ultraleap_v2/ulv2_interface.h + ) + add_library(drv_ulv2 STATIC ${ULV2_SOURCE_FILES}) + target_link_libraries(drv_ulv2 PRIVATE xrt-interfaces aux_util aux_math LeapV2::LeapV2) +endif() + if(XRT_BUILD_DRIVER_OHMD) set(OHMD_SOURCE_FILES ohmd/oh_device.c @@ -97,18 +148,7 @@ if(XRT_BUILD_DRIVER_OHMD) list(APPEND ENABLED_HEADSET_DRIVERS openhmd) endif() -if(XRT_BUILD_DRIVER_HANDTRACKING) - set(HT_SOURCE_FILES - ht/ht_driver.c - ht/ht_driver.h - ht/ht_interface.h - ht/ht_prober.c - ) - add_library(drv_ht STATIC ${HT_SOURCE_FILES}) - target_link_libraries(drv_ht PRIVATE xrt-interfaces aux_util aux_math) - list(APPEND ENABLED_DRIVERS ht) -endif() if(XRT_BUILD_DRIVER_PSMV) set(PSMOVE_SOURCE_FILES @@ -138,7 +178,11 @@ endif() if(XRT_BUILD_DRIVER_RS) set(RS_SOURCE_FILES - realsense/rs_6dof.c + realsense/rs_ddev.c + realsense/rs_hdev.c + realsense/rs_prober.c + realsense/rs_driver.h + realsense/rs_interface.h ) add_library(drv_rs STATIC ${RS_SOURCE_FILES}) @@ -160,6 +204,7 @@ if(XRT_BUILD_DRIVER_REMOTE) list(APPEND ENABLED_HEADSET_DRIVERS remote) endif() +set(VIVE_CONFIG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/vive") if(XRT_BUILD_DRIVER_VIVE) set(VIVE_SOURCE_FILES vive/vive_device.h @@ -170,17 +215,19 @@ if(XRT_BUILD_DRIVER_VIVE) vive/vive_protocol.h vive/vive_controller.h vive/vive_controller.c - vive/vive_config.h - vive/vive_config.c vive/vive_lighthouse.h vive/vive_lighthouse.c ) add_library(drv_vive STATIC ${VIVE_SOURCE_FILES}) - target_link_libraries(drv_vive PRIVATE xrt-interfaces aux_os aux_util aux_math xrt-external-cjson) + target_link_libraries(drv_vive PRIVATE xrt-interfaces aux_os aux_util aux_math xrt-external-cjson aux_vive) target_link_libraries(drv_vive PRIVATE ${ZLIB_LIBRARIES}) target_include_directories(drv_vive PRIVATE ${ZLIB_INCLUDE_DIRS}) list(APPEND ENABLED_HEADSET_DRIVERS vive) + + if (XRT_BUILD_DRIVER_HANDTRACKING) + target_link_libraries(drv_vive PRIVATE drv_ht) + endif() endif() if(XRT_HAVE_V4L2) @@ -189,33 +236,57 @@ if(XRT_HAVE_V4L2) ) add_library(drv_v4l2 STATIC ${V4L2_SOURCE_FILES}) - target_link_libraries(drv_v4l2 PRIVATE xrt-interfaces aux_os) + target_link_libraries(drv_v4l2 PRIVATE + xrt-interfaces + aux_os + aux_util + ) list(APPEND ENABLED_DRIVERS v4l2) endif() -if(XRT_HAVE_VF) +if(XRT_BUILD_DRIVER_VF) set(VF_SOURCE_FILES vf/vf_driver.c ) add_library(drv_vf STATIC ${VF_SOURCE_FILES}) - target_link_libraries(drv_vf PRIVATE xrt-interfaces aux_os ${GST_LIBRARIES}) + target_link_libraries(drv_vf PRIVATE xrt-interfaces aux_os aux_util ${GST_LIBRARIES}) target_include_directories(drv_vf PRIVATE ${GST_INCLUDE_DIRS}) - MESSAGE("GST include ${GST_INCLUDE_DIRS}") list(APPEND ENABLED_DRIVERS vf) endif() +if(XRT_BUILD_DRIVER_HANDTRACKING) + set(HT_SOURCE_FILES + ht/ht_driver.cpp + ht/ht_driver.hpp + ht/ht_interface.h + ht/ht_models.hpp + ht/ht_hand_math.hpp + ht/ht_image_math.hpp + ht/ht_nms.hpp + ht/templates/NaivePermutationSort.hpp + ) + add_library(drv_ht STATIC ${HT_SOURCE_FILES}) + target_link_libraries(drv_ht PRIVATE xrt-interfaces aux_os aux_util aux_math aux_gstreamer ONNXRuntime::ONNXRuntime ${OpenCV_LIBRARIES}) + target_include_directories(drv_ht PRIVATE ${OpenCV_INCLUDE_DIRS} ${EIGEN3_INCLUDE_DIR}) + list(APPEND ENABLED_DRIVERS ht) +endif() + if (XRT_BUILD_DRIVER_SURVIVE) set(SURVIVE_SOURCE_FILES survive/survive_driver.c + survive/survive_driver.h survive/survive_interface.h - survive/survive_wrap.c - survive/survive_wrap.h + survive/survive_prober.c ) add_library(drv_survive STATIC ${SURVIVE_SOURCE_FILES}) - target_link_libraries(drv_survive PRIVATE xrt-interfaces aux_os aux_util aux_math PkgConfig::SURVIVE) + target_link_libraries(drv_survive PRIVATE xrt-interfaces aux_os aux_util aux_math aux_vive PkgConfig::SURVIVE) list(APPEND ENABLED_HEADSET_DRIVERS survive) + + if (XRT_BUILD_DRIVER_HANDTRACKING) + target_link_libraries(drv_survive PRIVATE drv_ht) + endif() endif() if(XRT_BUILD_DRIVER_ANDROID) @@ -243,10 +314,49 @@ if (XRT_BUILD_DRIVER_ILLIXR) add_library(drv_illixr STATIC ${ILLIXR_SOURCE_FILES}) target_link_libraries(drv_illixr PUBLIC ${CMAKE_DL_LIBS} xrt-interfaces aux_util aux_os) target_include_directories(drv_illixr PUBLIC ${ILLIXR_PATH}) - target_compile_options(drv_illixr PUBLIC $<$:-std=c++17>) list(APPEND ENABLED_HEADSET_DRIVERS illixr) endif() +set(MUlTI_SOURCE_FILES + multi_wrapper/multi.c + multi_wrapper/multi.h + ) +add_library(drv_multi STATIC ${MUlTI_SOURCE_FILES}) +target_link_libraries(drv_multi PUBLIC xrt-interfaces aux_util) +list(APPEND ENABLED_HEADSET_DRIVERS drv_multi) + +if(XRT_BUILD_DRIVER_WMR) + set(WMR_SOURCE_FILES + wmr/wmr_common.h + wmr/wmr_config.c + wmr/wmr_config.h + wmr/wmr_hmd.c + wmr/wmr_hmd.h + wmr/wmr_interface.h + wmr/wmr_prober.c + wmr/wmr_protocol.c + wmr/wmr_protocol.h + ) + + add_library(drv_wmr STATIC ${WMR_SOURCE_FILES}) + target_link_libraries(drv_wmr PRIVATE xrt-interfaces aux_util aux_math xrt-external-cjson) + list(APPEND ENABLED_HEADSET_DRIVERS wmr) +endif() + +if(XRT_BUILD_DRIVER_EUROC) + set(EUROC_SOURCE_FILES + euroc/euroc_player.cpp + euroc/euroc_driver.h + euroc/euroc_device.c + euroc/euroc_interface.h + ) + + add_library(drv_euroc STATIC ${EUROC_SOURCE_FILES}) + target_link_libraries(drv_euroc PRIVATE xrt-interfaces aux_util aux_tracking ${OpenCV_LIBRARIES}) + target_include_directories(drv_euroc PRIVATE ${OpenCV_INCLUDE_DIRS}) + list(APPEND ENABLED_DRIVERS euroc) +endif() + if(ENABLED_HEADSET_DRIVERS) set(ENABLED_DRIVERS ${ENABLED_HEADSET_DRIVERS} ${ENABLED_DRIVERS}) list(SORT ENABLED_DRIVERS) @@ -254,4 +364,4 @@ if(ENABLED_HEADSET_DRIVERS) message(STATUS "Enabled drivers: ${ENABLED_DRIVERS}") else() message(FATAL_ERROR "You must enable at least one headset driver to build Monado.") -endif() \ No newline at end of file +endif() diff --git a/src/xrt/drivers/android/android_prober.c b/src/xrt/drivers/android/android_prober.c index 3830ed3b2..1057f8345 100644 --- a/src/xrt/drivers/android/android_prober.c +++ b/src/xrt/drivers/android/android_prober.c @@ -62,11 +62,16 @@ android_prober_destroy(struct xrt_auto_prober *p) } //! @public @memberof android_prober -static struct xrt_device * -android_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no_hmds, struct xrt_prober *xp) +static int +android_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) { struct android_device *dd = android_device_create(); - return &dd->base; + out_xdevs[0] = &dd->base; + return 1; } diff --git a/src/xrt/drivers/android/android_sensors.c b/src/xrt/drivers/android/android_sensors.c index bad38b5b7..57f0cd6d2 100644 --- a/src/xrt/drivers/android/android_sensors.c +++ b/src/xrt/drivers/android/android_sensors.c @@ -170,29 +170,12 @@ android_device_get_tracked_pose(struct xrt_device *xdev, static void android_device_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { - struct xrt_pose pose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; - bool adjust = view_index == 0; - - pose.position.x = eye_relation->x / 2.0f; - pose.position.y = eye_relation->y / 2.0f; - pose.position.z = eye_relation->z / 2.0f; - - // Adjust for left/right while also making sure there aren't any -0.f. - if (pose.position.x > 0.0f && adjust) { - pose.position.x = -pose.position.x; - } - if (pose.position.y > 0.0f && adjust) { - pose.position.y = -pose.position.y; - } - if (pose.position.z > 0.0f && adjust) { - pose.position.z = -pose.position.z; - } - - *out_pose = pose; + (void)xdev; + u_device_get_view_pose(eye_relation, view_index, out_pose); } /* @@ -225,6 +208,7 @@ android_device_create() d->base.inputs[0].name = XRT_INPUT_GENERIC_HEAD_POSE; d->base.device_type = XRT_DEVICE_TYPE_HMD; snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "Android Sensors"); + snprintf(d->base.serial, XRT_DEVICE_NAME_LEN, "Android Sensors"); d->ll = debug_get_log_option_android_log(); diff --git a/src/xrt/drivers/arduino/arduino_device.c b/src/xrt/drivers/arduino/arduino_device.c index 125764a14..53a2c5242 100644 --- a/src/xrt/drivers/arduino/arduino_device.c +++ b/src/xrt/drivers/arduino/arduino_device.c @@ -390,6 +390,10 @@ arduino_device_create(struct os_ble_device *ble) ad->base.binding_profiles = binding_profiles; ad->base.num_binding_profiles = ARRAY_SIZE(binding_profiles); + static int controller_num = 0; + snprintf(ad->base.str, XRT_DEVICE_NAME_LEN, "Arduino"); + snprintf(ad->base.serial, XRT_DEVICE_NAME_LEN, "Arduino %d", controller_num++); + ad->ble = ble; ad->ll = debug_get_log_option_arduino_log(); diff --git a/src/xrt/drivers/arduino/arduino_prober.c b/src/xrt/drivers/arduino/arduino_prober.c index 158266544..fd8fdfc89 100644 --- a/src/xrt/drivers/arduino/arduino_prober.c +++ b/src/xrt/drivers/arduino/arduino_prober.c @@ -66,12 +66,16 @@ arduino_prober_destroy(struct xrt_auto_prober *p) } //! @public @memberof arduino_prober -static struct xrt_device * -arduino_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no_hmds, struct xrt_prober *xp) +static int +arduino_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) { struct arduino_prober *ap = arduino_prober(xap); if (!ap->enabled) { - return NULL; + return 0; } const char *dev_uuid = "00004242-0000-1000-8000-004242424242"; @@ -80,10 +84,11 @@ arduino_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool struct os_ble_device *ble = NULL; os_ble_notify_open(dev_uuid, char_uuid, &ble); if (ble == NULL) { - return NULL; + return 0; } - return arduino_device_create(ble); + out_xdevs[0] = arduino_device_create(ble); + return 1; } diff --git a/src/xrt/drivers/daydream/daydream_device.c b/src/xrt/drivers/daydream/daydream_device.c index 0fc221cf6..3e6ac9dcc 100644 --- a/src/xrt/drivers/daydream/daydream_device.c +++ b/src/xrt/drivers/daydream/daydream_device.c @@ -370,6 +370,10 @@ daydream_device_create(struct os_ble_device *ble) dd->base.binding_profiles = binding_profiles; dd->base.num_binding_profiles = ARRAY_SIZE(binding_profiles); + static int controller_num = 0; + snprintf(dd->base.str, XRT_DEVICE_NAME_LEN, "Daydream"); + snprintf(dd->base.serial, XRT_DEVICE_NAME_LEN, "Daydream %d", controller_num++); + dd->ble = ble; dd->ll = debug_get_log_option_daydream_log(); diff --git a/src/xrt/drivers/daydream/daydream_prober.c b/src/xrt/drivers/daydream/daydream_prober.c index 7d65aa5f9..989d18ea4 100644 --- a/src/xrt/drivers/daydream/daydream_prober.c +++ b/src/xrt/drivers/daydream/daydream_prober.c @@ -66,12 +66,16 @@ daydream_prober_destroy(struct xrt_auto_prober *p) } //! @public @memberof daydream_prober -static struct xrt_device * -daydream_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no_hmds, struct xrt_prober *xp) +static int +daydream_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) { struct daydream_prober *pdaydream = daydream_prober(xap); if (!pdaydream->enabled) { - return NULL; + return 0; } const char *dev_uuid = "0000fe55-0000-1000-8000-00805f9b34fb"; @@ -80,12 +84,13 @@ daydream_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, boo struct os_ble_device *ble = NULL; os_ble_notify_open(dev_uuid, char_uuid, &ble); if (ble == NULL) { - return NULL; + return 0; } struct daydream_device *dd = daydream_device_create(ble); - return &dd->base; + out_xdevs[0] = &dd->base; + return 1; } diff --git a/src/xrt/drivers/depthai/depthai_driver.cpp b/src/xrt/drivers/depthai/depthai_driver.cpp new file mode 100644 index 000000000..5f09f81f0 --- /dev/null +++ b/src/xrt/drivers/depthai/depthai_driver.cpp @@ -0,0 +1,451 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief DepthAI frameserver implementation. + * @author Moses Turner + * @author Jakob Bornecrantz + * @ingroup drv_depthai + */ + +#include "os/os_time.h" +#include "os/os_threading.h" + +#include "util/u_var.h" +#include "util/u_misc.h" +#include "util/u_debug.h" +#include "util/u_frame.h" +#include "util/u_format.h" +#include "util/u_logging.h" +#include "util/u_trace_marker.h" + +#include "depthai_interface.h" + +#include "depthai/depthai.hpp" + +#include +#include +#include +#include + +#include +#include + + +/* + * + * Printing functions. + * + */ + +#define DEPTHAI_TRACE(d, ...) U_LOG_IFL_T(d->ll, __VA_ARGS__) +#define DEPTHAI_DEBUG(d, ...) U_LOG_IFL_D(d->ll, __VA_ARGS__) +#define DEPTHAI_INFO(d, ...) U_LOG_IFL_I(d->ll, __VA_ARGS__) +#define DEPTHAI_WARN(d, ...) U_LOG_IFL_W(d->ll, __VA_ARGS__) +#define DEPTHAI_ERROR(d, ...) U_LOG_IFL_E(d->ll, __VA_ARGS__) + +DEBUG_GET_ONCE_LOG_OPTION(depthai_log, "DEPTHAI_LOG", U_LOGGING_WARN) + + +/* + * + * Helper frame wrapper code. + * + */ + +class DepthAIFrameWrapper +{ +public: + struct xrt_frame frame = {}; + + std::shared_ptr depthai_frame = {}; +}; + +extern "C" void +depthai_frame_wrapper_destroy(struct xrt_frame *xf) +{ + DepthAIFrameWrapper *dfw = (DepthAIFrameWrapper *)xf; + delete dfw; +} + + +/* + * + * DepthAI frameserver. + * + */ + +enum depthai_camera_type +{ + RGB_IMX_378, + RGB_OV_9782, + MONO_OV_9282_L, + MONO_OV_9282_R, +}; + +struct depthai_fs +{ + struct xrt_fs base; + struct xrt_frame_node node; + struct os_thread_helper play_thread; + + u_logging_level ll; + + uint32_t width; + uint32_t height; + xrt_format format; + + xrt_frame_sink *sink; + + dai::Device *device; + dai::DataOutputQueue *queue; + + dai::ColorCameraProperties::SensorResolution color_sensor_resoultion; + dai::ColorCameraProperties::ColorOrder color_order; + + dai::MonoCameraProperties::SensorResolution mono_sensor_resoultion; + dai::CameraBoardSocket camera_board_socket; + + dai::CameraImageOrientation image_orientation; + uint32_t fps; + bool interleaved; +}; + + +/* + * + * Internal functions. + * + */ + +static void +depthai_print_connected_cameras(struct depthai_fs *depthai) +{ + std::ostringstream oss = {}; + for (const auto &cam : depthai->device->getConnectedCameras()) { + oss << "'" << static_cast(cam) << "' "; + } + std::string str = oss.str(); + + DEPTHAI_DEBUG(depthai, "DepthAI: Connected cameras: %s", str.c_str()); +} + +static void +depthai_do_one_frame(struct depthai_fs *depthai) +{ + std::shared_ptr imgFrame = depthai->queue->get(); + if (!imgFrame) { + std::cout << "Not ImgFrame" << std::endl; + return; // Nothing to do. + } + + // Trace-marker here for timing after we have gotten a frame. + SINK_TRACE_IDENT(depthai_frame); + + // Get the timestamp. + auto duration = imgFrame->getTimestamp().time_since_epoch(); + auto nano = std::chrono::duration_cast>(duration); + uint64_t timestamp_ns = nano.count(); + + // Create a wrapper that will keep the frame alive as long as the frame was alive. + DepthAIFrameWrapper *dfw = new DepthAIFrameWrapper(); + dfw->depthai_frame = imgFrame; + + // Fill in all of the data. + struct xrt_frame *xf = &dfw->frame; + xf->reference.count = 1; + xf->destroy = depthai_frame_wrapper_destroy; + xf->width = depthai->width; + xf->height = depthai->height; + xf->format = depthai->format; + xf->timestamp = timestamp_ns; + xf->data = imgFrame->getData().data(); + + // Calculate stride and size, assuming tightly packed rows. + u_format_size_for_dimensions(xf->format, xf->width, xf->height, &xf->stride, &xf->size); + + // Push the frame to the sink. + xrt_sink_push_frame(depthai->sink, xf); + + // If downstream wants to keep the frame they would have referenced it. + xrt_frame_reference(&xf, NULL); +} + +static void * +depthai_mainloop(void *ptr) +{ + SINK_TRACE_MARKER(); + + struct depthai_fs *depthai = (struct depthai_fs *)ptr; + DEPTHAI_DEBUG(depthai, "DepthAI: Mainloop called"); + + os_thread_helper_lock(&depthai->play_thread); + while (os_thread_helper_is_running_locked(&depthai->play_thread)) { + os_thread_helper_unlock(&depthai->play_thread); + + depthai_do_one_frame(depthai); + + // Need to lock the thread when we go back to the while condition. + os_thread_helper_lock(&depthai->play_thread); + } + os_thread_helper_unlock(&depthai->play_thread); + + DEPTHAI_DEBUG(depthai, "DepthAI: Mainloop exiting"); + return nullptr; +} + +static bool +depthai_destroy(struct depthai_fs *depthai) +{ + DEPTHAI_DEBUG(depthai, "DepthAI: Frameserver destroy called"); + + os_thread_helper_destroy(&depthai->play_thread); + + delete depthai->device; + + free(depthai); + + return true; +} + + +/* + * + * Frame server functions. + * + */ + +/*! + * Cast to derived type. + */ +static inline struct depthai_fs * +depthai_fs(struct xrt_fs *xfs) +{ + return (struct depthai_fs *)xfs; +} + +static bool +depthai_fs_enumerate_modes(struct xrt_fs *xfs, struct xrt_fs_mode **out_modes, uint32_t *out_count) +{ + struct depthai_fs *depthai = depthai_fs(xfs); + DEPTHAI_DEBUG(depthai, "DepthAI: Enumerate modes called"); + + struct xrt_fs_mode *modes = U_TYPED_ARRAY_CALLOC(struct xrt_fs_mode, 1); + if (modes == NULL) { + return false; + } + + modes[0].width = depthai->width; + modes[0].height = depthai->height; + modes[0].format = depthai->format; + modes[0].stereo_format = XRT_STEREO_FORMAT_NONE; + + *out_modes = modes; + *out_count = 1; + + return true; +} + +static bool +depthai_fs_configure_capture(struct xrt_fs *xfs, struct xrt_fs_capture_parameters *cp) +{ + struct depthai_fs *depthai = depthai_fs(xfs); + DEPTHAI_DEBUG(depthai, "DepthAI: Configure capture called"); + + // Noop + return false; +} + +static bool +depthai_fs_stream_start(struct xrt_fs *xfs, + struct xrt_frame_sink *xs, + enum xrt_fs_capture_type capture_type, + uint32_t descriptor_index) +{ + struct depthai_fs *depthai = depthai_fs(xfs); + DEPTHAI_DEBUG(depthai, "DepthAI: Stream start called"); + + assert(descriptor_index == 0); + (void)capture_type; // Don't care about this one just yet. + + depthai->sink = xs; + + os_thread_helper_start(&depthai->play_thread, depthai_mainloop, depthai); + + return true; +} + +static bool +depthai_fs_stream_stop(struct xrt_fs *xfs) +{ + struct depthai_fs *depthai = depthai_fs(xfs); + DEPTHAI_DEBUG(depthai, "DepthAI: Stream stop called"); + + // This call fully stops the thread. + os_thread_helper_stop(&depthai->play_thread); + + return true; +} + +static bool +depthai_fs_is_running(struct xrt_fs *xfs) +{ + struct depthai_fs *depthai = depthai_fs(xfs); + + os_thread_helper_lock(&depthai->play_thread); + bool running = os_thread_helper_is_running_locked(&depthai->play_thread); + os_thread_helper_unlock(&depthai->play_thread); + + return running; +} + + +/* + * + * Node functions. + * + */ + +static void +depthai_fs_node_break_apart(struct xrt_frame_node *node) +{ + struct depthai_fs *depthai = container_of(node, struct depthai_fs, node); + DEPTHAI_DEBUG(depthai, "DepthAI: Node break apart called"); + + depthai_fs_stream_stop(&depthai->base); +} + +static void +depthai_fs_node_destroy(struct xrt_frame_node *node) +{ + struct depthai_fs *depthai = container_of(node, struct depthai_fs, node); + DEPTHAI_DEBUG(depthai, "DepthAI: Node destroy called"); + + // Safe to call, break apart have already stopped the stream. + depthai_destroy(depthai); +} + + +/* + * + * 'Exported' functions. + * + */ + +extern "C" struct xrt_fs * +depthai_fs_single_rgb(struct xrt_frame_context *xfctx) +{ + // Try to create a device and see if that fail first. + dai::Device *d; + try { + d = new dai::Device(); + } catch (std::exception &e) { + std::string what = e.what(); + U_LOG_E("DepthAI error: %s", what.c_str()); + return nullptr; + } + + struct depthai_fs *depthai = U_TYPED_CALLOC(struct depthai_fs); + depthai->base.enumerate_modes = depthai_fs_enumerate_modes; + depthai->base.configure_capture = depthai_fs_configure_capture; + depthai->base.stream_start = depthai_fs_stream_start; + depthai->base.stream_stop = depthai_fs_stream_stop; + depthai->base.is_running = depthai_fs_is_running; + depthai->node.break_apart = depthai_fs_node_break_apart; + depthai->node.destroy = depthai_fs_node_destroy; + depthai->ll = debug_get_log_option_depthai_log(); + depthai->device = d; + + enum depthai_camera_type camera_type = RGB_OV_9782; + + switch (camera_type) { + case (RGB_OV_9782): + depthai->width = 1280; + depthai->height = 800; + depthai->format = XRT_FORMAT_R8G8B8; + depthai->color_sensor_resoultion = dai::ColorCameraProperties::SensorResolution::THE_800_P; + depthai->image_orientation = dai::CameraImageOrientation::ROTATE_180_DEG; + depthai->fps = 60; // Currently only supports 60. + depthai->interleaved = true; + depthai->color_order = dai::ColorCameraProperties::ColorOrder::RGB; + break; + case (RGB_IMX_378): + depthai->width = 1920; + depthai->height = 1080; + depthai->format = XRT_FORMAT_R8G8B8; + depthai->color_sensor_resoultion = dai::ColorCameraProperties::SensorResolution::THE_1080_P; + depthai->image_orientation = dai::CameraImageOrientation::AUTO; + depthai->fps = 118; // Actual max. + depthai->interleaved = true; + depthai->color_order = dai::ColorCameraProperties::ColorOrder::RGB; + break; + case (MONO_OV_9282_L): + depthai->width = 1280; + depthai->height = 800; + depthai->format = XRT_FORMAT_L8; + depthai->camera_board_socket = dai::CameraBoardSocket::LEFT; + depthai->mono_sensor_resoultion = dai::MonoCameraProperties::SensorResolution::THE_800_P; + depthai->image_orientation = dai::CameraImageOrientation::AUTO; + depthai->fps = 60; // Currently only supports 60. + break; + case (MONO_OV_9282_R): + depthai->width = 1280; + depthai->height = 800; + depthai->format = XRT_FORMAT_L8; + depthai->camera_board_socket = dai::CameraBoardSocket::RIGHT; + depthai->mono_sensor_resoultion = dai::MonoCameraProperties::SensorResolution::THE_800_P; + depthai->image_orientation = dai::CameraImageOrientation::AUTO; + depthai->fps = 60; // Currently only supports 60. + break; + default: assert(false); + } + + // Some debug printing. + depthai_print_connected_cameras(depthai); + + // Make sure that the thread helper is initialised. + os_thread_helper_init(&depthai->play_thread); + + dai::Pipeline p = {}; + + auto xlinkOut = p.create(); + xlinkOut->setStreamName("preview"); + + std::shared_ptr colorCam = nullptr; + std::shared_ptr monoCam = nullptr; + + if (depthai->format == XRT_FORMAT_R8G8B8) { + colorCam = p.create(); + colorCam->setPreviewSize(depthai->width, depthai->height); + colorCam->setResolution(depthai->color_sensor_resoultion); + colorCam->setImageOrientation(depthai->image_orientation); + colorCam->setInterleaved(depthai->interleaved); + colorCam->setFps(depthai->fps); + colorCam->setColorOrder(depthai->color_order); + + // Link plugins CAM -> XLINK + colorCam->preview.link(xlinkOut->input); + } + + if (depthai->format == XRT_FORMAT_L8) { + monoCam = p.create(); + monoCam->setBoardSocket(depthai->camera_board_socket); + monoCam->setResolution(depthai->mono_sensor_resoultion); + monoCam->setImageOrientation(depthai->image_orientation); + monoCam->setFps(depthai->fps); + + // Link plugins CAM -> XLINK + monoCam->out.link(xlinkOut->input); + } + + + // Start the pipeline + d->startPipeline(p); + depthai->queue = d->getOutputQueue("preview", 1, false).get(); // out of shared pointer + + xrt_frame_context_add(xfctx, &depthai->node); + + DEPTHAI_DEBUG(depthai, "DepthAI: Created"); + + return &depthai->base; +} diff --git a/src/xrt/drivers/depthai/depthai_interface.h b/src/xrt/drivers/depthai/depthai_interface.h new file mode 100644 index 000000000..79aadc18f --- /dev/null +++ b/src/xrt/drivers/depthai/depthai_interface.h @@ -0,0 +1,38 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Interface header for DepthAI camera. + * @author Moses Turner + * @author Jakob Bornecrantz + * @ingroup drv_depthai + */ + +#pragma once + +#include "xrt/xrt_frameserver.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +/*! + * @defgroup drv_depthai DepthAI frameserver driver + * @ingroup drv + * + * @brief Frameserver for the DepthAI camera module. + */ + +/*! + * Create a DepthAI frameserver using a single RGB camera. + * + * @ingroup drv_depthai + */ +struct xrt_fs * +depthai_fs_single_rgb(struct xrt_frame_context *xfctx); + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/dummy/dummy_hmd.c b/src/xrt/drivers/dummy/dummy_hmd.c index babe5bb68..8091ea39e 100644 --- a/src/xrt/drivers/dummy/dummy_hmd.c +++ b/src/xrt/drivers/dummy/dummy_hmd.c @@ -31,6 +31,13 @@ * */ +enum dummy_movement +{ + DUMMY_WOBBLE, + DUMMY_ROTATE, +}; + + /*! * A example HMD device. * @@ -47,6 +54,7 @@ struct dummy_hmd float diameter_m; enum u_logging_level log_level; + enum dummy_movement movement; }; @@ -63,6 +71,7 @@ dummy_hmd(struct xrt_device *xdev) } DEBUG_GET_ONCE_LOG_OPTION(dummy_log, "DUMMY_LOG", U_LOGGING_WARN) +DEBUG_GET_ONCE_BOOL_OPTION(dummy_rotate, "DUMMY_ROTATE", false) #define DH_TRACE(p, ...) U_LOG_XDEV_IFL_T(&dh->base, dh->log_level, __VA_ARGS__) #define DH_DEBUG(p, ...) U_LOG_XDEV_IFL_D(&dh->base, dh->log_level, __VA_ARGS__) @@ -98,20 +107,35 @@ dummy_hmd_get_tracked_pose(struct xrt_device *xdev, return; } - double time_s = time_ns_to_s(at_timestamp_ns - dh->created_ns); - double d = dh->diameter_m; - double d2 = d * 2; - double t = 2.0; - double t2 = t * 2; - double t3 = t * 3; - double t4 = t * 4; - dh->pose.position.x = dh->center.x + sin((time_s / t2) * M_PI) * d2 - d; - dh->pose.position.y = dh->center.y + sin((time_s / t) * M_PI) * d; - dh->pose.orientation.x = sin((time_s / t3) * M_PI) / 64.0; - dh->pose.orientation.y = sin((time_s / t4) * M_PI) / 16.0; - dh->pose.orientation.z = sin((time_s / t4) * M_PI) / 64.0; - dh->pose.orientation.w = 1; - math_quat_normalize(&dh->pose.orientation); + const double time_s = time_ns_to_s(at_timestamp_ns - dh->created_ns); + const double d = dh->diameter_m; + const double d2 = d * 2; + const double t = 2.0; + const double t2 = t * 2; + const double t3 = t * 3; + const double t4 = t * 4; + const struct xrt_vec3 up = {0, 1, 0}; + + switch (dh->movement) { + default: + case DUMMY_WOBBLE: + // Wobble time. + dh->pose.position.x = dh->center.x + sin((time_s / t2) * M_PI) * d2 - d; + dh->pose.position.y = dh->center.y + sin((time_s / t) * M_PI) * d; + dh->pose.orientation.x = sin((time_s / t3) * M_PI) / 64.0; + dh->pose.orientation.y = sin((time_s / t4) * M_PI) / 16.0; + dh->pose.orientation.z = sin((time_s / t4) * M_PI) / 64.0; + dh->pose.orientation.w = 1; + math_quat_normalize(&dh->pose.orientation); + break; + case DUMMY_ROTATE: + // Reset position. + dh->pose.position = dh->center; + + // Rotate around the up vector. + math_quat_from_angle_vector(time_s / 4, &up, &dh->pose.orientation); + break; + } out_relation->pose = dh->pose; out_relation->relation_flags = (enum xrt_space_relation_flags)(XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | @@ -121,29 +145,12 @@ dummy_hmd_get_tracked_pose(struct xrt_device *xdev, static void dummy_hmd_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { - struct xrt_pose pose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; - bool adjust = view_index == 0; - - pose.position.x = eye_relation->x / 2.0f; - pose.position.y = eye_relation->y / 2.0f; - pose.position.z = eye_relation->z / 2.0f; - - // Adjust for left/right while also making sure there aren't any -0.f. - if (pose.position.x > 0.0f && adjust) { - pose.position.x = -pose.position.x; - } - if (pose.position.y > 0.0f && adjust) { - pose.position.y = -pose.position.y; - } - if (pose.position.z > 0.0f && adjust) { - pose.position.z = -pose.position.z; - } - - *out_pose = pose; + (void)xdev; + u_device_get_view_pose(eye_relation, view_index, out_pose); } struct xrt_device * @@ -165,6 +172,7 @@ dummy_hmd_create(void) // Print name. snprintf(dh->base.str, XRT_DEVICE_NAME_LEN, "Dummy HMD"); + snprintf(dh->base.serial, XRT_DEVICE_NAME_LEN, "Dummy HMD"); // Setup input. dh->base.inputs[0].name = XRT_INPUT_GENERIC_HEAD_POSE; @@ -186,6 +194,12 @@ dummy_hmd_create(void) return NULL; } + // Select the type of movement. + dh->movement = DUMMY_WOBBLE; + if (debug_get_bool_option_dummy_rotate()) { + dh->movement = DUMMY_ROTATE; + } + // Setup variable tracker. u_var_add_root(dh, "Dummy HMD", true); u_var_add_pose(dh, &dh->pose, "pose"); diff --git a/src/xrt/drivers/dummy/dummy_prober.c b/src/xrt/drivers/dummy/dummy_prober.c index 29eaf3880..09676fbd9 100644 --- a/src/xrt/drivers/dummy/dummy_prober.c +++ b/src/xrt/drivers/dummy/dummy_prober.c @@ -42,18 +42,23 @@ dummy_prober_destroy(struct xrt_auto_prober *p) } //! @public @memberof dummy_prober -static struct xrt_device * -dummy_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no_hmds, struct xrt_prober *xp) +static int +dummy_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) { struct dummy_prober *dp = dummy_prober(xap); (void)dp; // Do not create a dummy HMD if we are not looking for HMDs. if (no_hmds) { - return NULL; + return 0; } - return dummy_hmd_create(); + out_xdevs[0] = dummy_hmd_create(); + return 1; } struct xrt_auto_prober * diff --git a/src/xrt/drivers/euroc/euroc_device.c b/src/xrt/drivers/euroc/euroc_device.c new file mode 100644 index 000000000..a2d9c5db3 --- /dev/null +++ b/src/xrt/drivers/euroc/euroc_device.c @@ -0,0 +1,257 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Fake device tracked with EuRoC datasets and SLAM. + * @author Mateo de Mayo + * @ingroup drv_euroc + */ + +#include "util/u_misc.h" +#include "util/u_debug.h" +#include "util/u_device.h" +#include "util/u_distortion_mesh.h" +#include "util/u_var.h" +#include "math/m_space.h" +#include "math/m_mathinclude.h" +#include "xrt/xrt_prober.h" +#include "xrt/xrt_tracking.h" +#include "xrt/xrt_config_have.h" + +#include "euroc_driver.h" + +#include + +DEBUG_GET_ONCE_BOOL_OPTION(euroc_hmd, "EUROC_HMD", false) + +struct xrt_device * +euroc_device_create(struct xrt_prober *xp); + +// Euroc Device Prober + +/*! + * @implements xrt_auto_prober + */ +struct euroc_prober +{ + struct xrt_auto_prober base; +}; + +static inline struct euroc_prober * +euroc_prober(struct xrt_auto_prober *p) +{ + return (struct euroc_prober *)p; +} + +static void +euroc_prober_destroy(struct xrt_auto_prober *p) +{ + struct euroc_prober *epr = euroc_prober(p); + free(epr); +} + +static int +euroc_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) +{ + struct euroc_prober *epr = euroc_prober(xap); + (void)epr; + + struct xrt_device *xd = euroc_device_create(xp); + if (xd == NULL) { + return 0; + } + + out_xdevs[0] = xd; + return 1; +} + +struct xrt_auto_prober * +euroc_create_auto_prober() +{ + // `ep` var name used for euroc_player, let's use `epr` instead + struct euroc_prober *epr = U_TYPED_CALLOC(struct euroc_prober); + epr->base.name = "Euroc Device"; + epr->base.destroy = euroc_prober_destroy; + epr->base.lelo_dallas_autoprobe = euroc_prober_autoprobe; + return &epr->base; +} + + +// Euroc Device + +struct euroc_device +{ + struct xrt_device base; + struct xrt_tracked_slam *slam; + struct xrt_pose offset; + struct xrt_pose pose; + struct xrt_tracking_origin tracking_origin; + enum u_logging_level ll; +}; + +static inline struct euroc_device * +euroc_device(struct xrt_device *xdev) +{ + return (struct euroc_device *)xdev; +} + +static void +euroc_device_update_inputs(struct xrt_device *xdev) +{ + return; +} + +//! Corrections specific for original euroc datasets and Kimera. +//! If your datasets comes from a different camera you should probably +//! use a different pose correction function. +XRT_MAYBE_UNUSED static inline struct xrt_pose +euroc_device_correct_pose_from_kimera(struct xrt_pose pose) +{ + //! @todo Implement proper pose corrections for the original euroc datasets + //! @todo Allow to use different pose corrections depending on the device used to record + return pose; +} + +static void +euroc_device_get_tracked_pose(struct xrt_device *xdev, + enum xrt_input_name name, + uint64_t at_timestamp_ns, + struct xrt_space_relation *out_relation) +{ + struct euroc_device *ed = euroc_device(xdev); + + if (ed->slam) { + EUROC_ASSERT_(at_timestamp_ns < INT64_MAX); + xrt_tracked_slam_get_tracked_pose(ed->slam, at_timestamp_ns, out_relation); + + int pose_bits = XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT; + bool pose_tracked = out_relation->relation_flags & pose_bits; + if (pose_tracked) { +#if defined(XRT_HAVE_KIMERA_SLAM) + ed->pose = euroc_device_correct_pose_from_kimera(out_relation->pose); +#else + ed->pose = out_relation->pose; +#endif + } + } + + struct xrt_space_graph space_graph = {0}; + m_space_graph_add_pose(&space_graph, &ed->pose); + m_space_graph_add_pose(&space_graph, &ed->offset); + m_space_graph_resolve(&space_graph, out_relation); + out_relation->relation_flags = (enum xrt_space_relation_flags)( + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_POSITION_VALID_BIT | + XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT); +} + +static void +euroc_get_view_pose(struct xrt_device *xdev, + const struct xrt_vec3 *eye_relation, + uint32_t view_index, + struct xrt_pose *out_pose) +{ + (void)xdev; + u_device_get_view_pose(eye_relation, view_index, out_pose); +} + +static void +euroc_device_destroy(struct xrt_device *xdev) +{ + struct euroc_device *ed = euroc_device(xdev); + u_var_remove_root(ed); + u_device_free(&ed->base); +} + +struct xrt_device * +euroc_device_create(struct xrt_prober *xp) +{ + const char *euroc_path = debug_get_option_euroc_path(); + if (euroc_path == NULL) { + return NULL; + } + + bool is_hmd = debug_get_bool_option_euroc_hmd(); + + enum u_device_alloc_flags flags = U_DEVICE_ALLOC_NO_FLAGS; + if (is_hmd) { + flags |= U_DEVICE_ALLOC_HMD | U_DEVICE_ALLOC_TRACKING_NONE; + } + + struct euroc_device *ed = U_DEVICE_ALLOCATE(struct euroc_device, flags, 1, 0); + EUROC_ASSERT(ed != NULL, "Unable to allocate device"); + + ed->pose = (struct xrt_pose){{0, 0, 0, 1}, {0, 0, 0}}; + ed->offset = (struct xrt_pose){{0, 0, 0, 1}, {0.2, 1.3, -0.5}}; + ed->ll = debug_get_log_option_euroc_log(); + + struct xrt_device *xd = &ed->base; + + const char *dev_name; + if (is_hmd) { + xd->name = XRT_DEVICE_GENERIC_HMD; + xd->device_type = XRT_DEVICE_TYPE_HMD; + dev_name = "Euroc HMD"; + } else { + xd->name = XRT_DEVICE_SIMPLE_CONTROLLER; + xd->device_type = XRT_DEVICE_TYPE_ANY_HAND_CONTROLLER; + dev_name = "Euroc Controller"; + } + + snprintf(xd->str, XRT_DEVICE_NAME_LEN, "%s", dev_name); + snprintf(xd->serial, XRT_DEVICE_NAME_LEN, "%s", dev_name); + + // Fill in xd->hmd + if (is_hmd) { + struct u_device_simple_info info; + info.display.w_pixels = 1280; + info.display.h_pixels = 720; + info.display.w_meters = 0.13f; + info.display.h_meters = 0.07f; + info.lens_horizontal_separation_meters = 0.13f / 2.0f; + info.lens_vertical_position_meters = 0.07f / 2.0f; + info.views[0].fov = 85.0f * (M_PI / 180.0f); + info.views[1].fov = 85.0f * (M_PI / 180.0f); + + bool ret = u_device_setup_split_side_by_side(xd, &info); + EUROC_ASSERT(ret, "Failed to setup HMD properties"); + + u_distortion_mesh_set_none(xd); + } + + xd->tracking_origin = &ed->tracking_origin; + xd->tracking_origin->type = XRT_TRACKING_TYPE_EXTERNAL_SLAM; + xd->tracking_origin->offset.orientation.w = 1.0f; + snprintf(xd->tracking_origin->name, XRT_TRACKING_NAME_LEN, "%s %s", dev_name, "SLAM Tracker"); + + if (is_hmd) { + xd->inputs[0].name = XRT_INPUT_GENERIC_HEAD_POSE; + } else { + xd->inputs[0].name = XRT_INPUT_SIMPLE_GRIP_POSE; + } + + xd->update_inputs = euroc_device_update_inputs; + xd->get_tracked_pose = euroc_device_get_tracked_pose; + xd->destroy = euroc_device_destroy; + if (is_hmd) { + xd->get_view_pose = euroc_get_view_pose; + } + + int ret = xp->tracking->create_tracked_slam(xp->tracking, xd, &ed->slam); + if (ret < 0) { + EUROC_WARN(ed, + "Unable to create the SLAM tracker so the Euroc device won't be tracked.\n\t" + "Did you provide the appropriate SLAM dependency when compiling?"); + // However, we can continue, maybe the user just wants to play with the euroc utilities + } + + u_var_add_root(ed, dev_name, false); + u_var_add_pose(ed, &ed->pose, "pose"); + u_var_add_pose(ed, &ed->offset, "offset"); + u_var_add_pose(ed, &ed->tracking_origin.offset, "tracking offset"); + + return xd; +} diff --git a/src/xrt/drivers/euroc/euroc_driver.h b/src/xrt/drivers/euroc/euroc_driver.h new file mode 100644 index 000000000..94357577d --- /dev/null +++ b/src/xrt/drivers/euroc/euroc_driver.h @@ -0,0 +1,49 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Internal header for the euroc driver utilities. + * @author Mateo de Mayo + * @ingroup drv_euroc + */ +#pragma once + +#include "util/u_logging.h" +#include "euroc_interface.h" +#include + +#define EUROC_TRACE(e, ...) U_LOG_IFL_T(e->ll, __VA_ARGS__) +#define EUROC_DEBUG(e, ...) U_LOG_IFL_D(e->ll, __VA_ARGS__) +#define EUROC_INFO(e, ...) U_LOG_IFL_I(e->ll, __VA_ARGS__) +#define EUROC_WARN(e, ...) U_LOG_IFL_W(e->ll, __VA_ARGS__) +#define EUROC_ERROR(e, ...) U_LOG_IFL_E(e->ll, __VA_ARGS__) +#define EUROC_ASSERT(predicate, ...) \ + do { \ + bool p = predicate; \ + if (!p) { \ + U_LOG(U_LOGGING_ERROR, __VA_ARGS__); \ + assert(false && "EUROC_ASSERT failed: " #predicate); \ + exit(EXIT_FAILURE); \ + } \ + } while (false); +#define EUROC_ASSERT_(predicate) EUROC_ASSERT(predicate, "Assertion failed " #predicate) + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * @addtogroup drv_euroc + * @{ + */ + +DEBUG_GET_ONCE_LOG_OPTION(euroc_log, "EUROC_LOG", U_LOGGING_WARN) +DEBUG_GET_ONCE_OPTION(euroc_path, "EUROC_PATH", NULL) + +/*! + * @} + */ + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/euroc/euroc_interface.h b/src/xrt/drivers/euroc/euroc_interface.h new file mode 100644 index 000000000..5b5f8586f --- /dev/null +++ b/src/xrt/drivers/euroc/euroc_interface.h @@ -0,0 +1,53 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Interface for @ref drv_euroc + * @author Mateo de Mayo + * @ingroup drv_euroc + */ + +#pragma once + +#include "xrt/xrt_frameserver.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * @defgroup drv_euroc Euroc driver + * @ingroup drv + * + * @brief Provide euroc dataset playback features for SLAM evaluation + * + * This driver works with any dataset using the EuRoC format. + * Original EuRoC datasets and more information about them can be found in + * https://projects.asl.ethz.ch/datasets/doku.php?id=kmavvisualinertialdatasets + */ + +/*! + * Create an euroc player from a path to a dataset. + * + * @ingroup drv_euroc + */ +struct xrt_fs * +euroc_player_create(struct xrt_frame_context *xfctx, const char *path); + +/*! + * Create a auto prober for the fake euroc device. + * + * @ingroup drv_euroc + */ +struct xrt_auto_prober * +euroc_create_auto_prober(void); + +/*! + * @dir drivers/euroc + * + * @brief @ref drv_euroc files. + */ + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/euroc/euroc_player.cpp b/src/xrt/drivers/euroc/euroc_player.cpp new file mode 100644 index 000000000..206c01ce0 --- /dev/null +++ b/src/xrt/drivers/euroc/euroc_player.cpp @@ -0,0 +1,763 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief EuRoC playback functionality + * @author Mateo de Mayo + * @ingroup drv_euroc + */ + +#include "xrt/xrt_tracking.h" +#include "xrt/xrt_frameserver.h" +#include "os/os_threading.h" +#include "util/u_debug.h" +#include "util/u_misc.h" +#include "util/u_var.h" +#include "util/u_sink.h" +#include "tracking/t_frame_cv_mat_wrapper.hpp" +#include "math/m_filter_fifo.h" + +#include "euroc_driver.h" + +#include +#include + +#define EUROC_PLAYER_STR "Euroc Player" +#define CLAMP(X, A, B) (MIN(MAX((X), (A)), (B))) + + +typedef std::pair img_sample; +typedef std::vector imu_samples; +typedef std::vector img_samples; + +enum euroc_player_ui_state +{ + UNINITIALIZED = 0, + NOT_STREAMING, + STREAM_PLAYING, + STREAM_PAUSED, + STREAM_ENDED +}; + +/*! + * Euroc player is in charge of the playback of a particular dataset. + * + * @implements xrt_fs + * @implements xrt_frame_node + * @implements xrt_imu_sink + */ +struct euroc_player +{ + struct xrt_fs base; + struct xrt_frame_node node; + + // Sinks + struct xrt_frame_sink left_sink; //!< Intermediate sink for left camera frames + struct xrt_frame_sink right_sink; //!< Intermediate sink for right camera frames + struct xrt_imu_sink imu_sink; //!< Intermediate sink for IMU samples + struct xrt_slam_sinks in_sinks; //!< Pointers to intermediate sinks + struct xrt_slam_sinks out_sinks; //!< Pointers to downstream sinks + + struct os_thread_helper play_thread; + enum u_logging_level ll; + struct xrt_fs_mode mode; //!< The only fs mode the euroc dataset provides + bool is_running; //!< Set only at start and stop of frameserver stream + + //! Contains information about the source dataset; set only at start + struct + { + char path[256]; + bool is_stereo; + bool is_colored; + uint32_t width; + uint32_t height; + } dataset; + + //! Next frame number to use, index in `left_imgs` and `right_imgs`. + //! Note that this expects that both cameras provide the same amount of frames. + //! Furthermore, it is also expected that their timestamps match. + uint64_t img_seq; + uint64_t imu_seq; //!< Next imu sample number to use, index in `imus` + imu_samples *imus; //!< List of all IMU samples read from the dataset + img_samples *left_imgs; //!< List of all image names to read from the dataset + img_samples *right_imgs; //!< List of all image names to read from the dataset + + // Timestamp correction fields (can be disabled through `use_source_ts`) + timepoint_ns base_ts; //!< First imu0 timestamp, samples timestamps are relative to this + timepoint_ns start_ts; //!< When did the dataset started to be played + timepoint_ns offset_ts; //!< Amount of ns to offset start_ns (pauses, skips, etc) + + //! Playback information. + //! Prefer to fill it before starting to push frames. Modifying them on + //! runtime will work with the debug sinks but probably not elsewhere + struct + { + bool stereo; //!< Whether to stream both left and right sinks or only left + bool color; //!< If RGB available but this is false, images will be loaded in grayscale + float skip_first_s; //!< Amount of initial seconds of the dataset to skip + float scale; //!< Scale of each frame; e.g., 0.5 (half), 1.0 (avoids resize) + double speed; //!< Intended reproduction speed, could be slower due to read times + bool send_all_imus_first; //!< If enabled all imu samples will be sent before img samples + bool paused; //!< Whether to pause the playback + bool use_source_ts; //!< If true, use the original timestamps from the dataset + } playback; + + // UI related fields + enum euroc_player_ui_state ui_state; + struct u_var_button start_btn; + struct u_var_button pause_btn; + char progress_text[128]; + struct u_sink_debug ui_left_sink; //!< Sink to display left frames in UI + struct u_sink_debug ui_right_sink; //!< Sink to display right frames in UI + struct m_ff_vec3_f32 *gyro_ff; //!< Used for displaying IMU data + struct m_ff_vec3_f32 *accel_ff; //!< Same as `gyro_ff` +}; + +static void +euroc_player_start_btn_cb(void *ptr); +static void +euroc_player_set_ui_state(struct euroc_player *ep, euroc_player_ui_state state); + + +// Euroc functionality + +//! Parse and load all IMU samples into `samples`, assumes data.csv is well formed +//! If `ep` is not null, will set `ep->base_ts` with the first timestamp read +//! If `read_n` > 0, read at most that amount of samples +//! Returns whether the appropriate data.csv file could be opened +static bool +euroc_player_preload_imu_data(struct euroc_player *ep, + std::string dataset_path, + imu_samples *samples, + int64_t read_n = -1) +{ + std::string csv_filename = dataset_path + "/mav0/imu0/data.csv"; + std::ifstream fin{csv_filename}; + if (!fin.is_open()) { + return false; + } + + std::string line; + std::getline(fin, line); // Skip header line + bool set_base_ts = ep != NULL; + + while (std::getline(fin, line) && read_n-- != 0) { + timepoint_ns timestamp; + double v[6]; + size_t i = 0; + size_t j = line.find(','); + timestamp = std::stoll(line.substr(i, j)); + for (size_t k = 0; k < 6; k++) { + i = j; + j = line.find(',', i + 1); + v[k] = std::stod(line.substr(i + 1, j)); + } + + // Save first IMU sample timestamp + if (set_base_ts) { + ep->base_ts = timestamp; + set_base_ts = false; + } + + xrt_imu_sample sample{timestamp, {v[3], v[4], v[5]}, {v[0], v[1], v[2]}}; + samples->push_back(sample); + } + return true; +} + +//! Parse and load image names and timestamps into `samples` +//! If read_n > 0, read at most that amount of samples +//! Returns whether the appropriate data.csv file could be opened +static bool +euroc_player_preload_img_data(std::string dataset_path, img_samples *samples, bool is_left, int64_t read_n = -1) +{ + // Parse image data, assumes data.csv is well formed + std::string cam_name = is_left ? "cam0" : "cam1"; + std::string imgs_path = dataset_path + "/mav0/" + cam_name + "/data"; + std::string csv_filename = dataset_path + "/mav0/" + cam_name + "/data.csv"; + std::ifstream fin{csv_filename}; + if (!fin.is_open()) { + return false; + } + + std::string line; + std::getline(fin, line); // Skip header line + while (std::getline(fin, line) && read_n-- != 0) { + size_t i = line.find(','); + timepoint_ns timestamp = std::stoll(line.substr(0, i)); + std::string img_name_tail = line.substr(i + 1); + + // Standard euroc datasets use CRLF line endings, so let's remove the extra '\r' + if (img_name_tail.back() == '\r') { + img_name_tail.pop_back(); + } + + std::string img_name = imgs_path + "/" + img_name_tail; + img_sample sample{timestamp, img_name}; + samples->push_back(sample); + } + return true; +} + +static void +euroc_player_preload(struct euroc_player *ep) +{ + ep->imus->clear(); + euroc_player_preload_imu_data(ep, ep->dataset.path, ep->imus); + + ep->left_imgs->clear(); + euroc_player_preload_img_data(ep->dataset.path, ep->left_imgs, true); + + if (ep->dataset.is_stereo) { + ep->right_imgs->clear(); + euroc_player_preload_img_data(ep->dataset.path, ep->right_imgs, false); + } +} + +//! Skips the first seconds of the dataset as specified by the user +static void +euroc_player_user_skip(struct euroc_player *ep) +{ + timepoint_ns skip_first_ns = ep->playback.skip_first_s * 1000 * 1000 * 1000; + + while (ep->imus->at(ep->imu_seq).timestamp_ns < skip_first_ns) { + ep->imu_seq++; + } + + while (ep->left_imgs->at(ep->img_seq).first < skip_first_ns) { + ep->img_seq++; + } + + ep->offset_ts -= skip_first_ns / ep->playback.speed; +} + +//! Determine and fill attributes of the dataset pointed by `path` +//! Assertion fails if `path` does not point to an euroc dataset +static void +euroc_player_fill_dataset_info(struct euroc_player *ep, const char *path) +{ + const char *euroc_path = debug_get_option_euroc_path(); + EUROC_ASSERT(strcmp(euroc_path, path) == 0, "Unexpected path=%s differs from EUROC_PATH=%s", path, euroc_path); + + snprintf(ep->dataset.path, sizeof(ep->dataset.path), "%s", path); + img_samples samples; + imu_samples _; + bool has_right_camera = euroc_player_preload_img_data(ep->dataset.path, &samples, false, 0); + bool has_left_camera = euroc_player_preload_img_data(ep->dataset.path, &samples, true, 1); + bool has_imu = euroc_player_preload_imu_data(NULL, ep->dataset.path, &_, 0); + bool is_valid_dataset = has_left_camera && has_imu; + EUROC_ASSERT(is_valid_dataset, "Invalid dataset %s", path); + + cv::Mat first_left_img = cv::imread(samples[0].second, cv::IMREAD_ANYCOLOR); + ep->dataset.is_stereo = has_right_camera; + ep->dataset.is_colored = first_left_img.channels() == 3; + ep->dataset.width = first_left_img.cols; + ep->dataset.height = first_left_img.rows; + EUROC_INFO(ep, "dataset information\n\tpath: %s\n\tis_stereo: %d, is_colored: %d, width: %d, height: %d", + ep->dataset.path, ep->dataset.is_stereo, ep->dataset.is_colored, ep->dataset.width, + ep->dataset.height); +} + + +// Playback functionality + +static struct euroc_player * +euroc_player(struct xrt_fs *xfs) +{ + return (struct euroc_player *)xfs; +} + +//! Wrapper around os_monotonic_get_ns to convert to int64_t and check ranges +static timepoint_ns +os_monotonic_get_ts() +{ + uint64_t uts = os_monotonic_get_ns(); + EUROC_ASSERT(uts < INT64_MAX, "Timestamp=%lu was greater than INT64_MAX=%ld", uts, INT64_MAX); + int64_t its = uts; + return its; +} + +//! @returns maps a timestamp to current time (wrt. ep->start_ts) +//! from the original euroc timestamp (uses first imu0 timestamp as base time) +static timepoint_ns +euroc_player_mapped_ts(struct euroc_player *ep, timepoint_ns ts) +{ + if (ep->playback.use_source_ts) { + return ts; + } + + timepoint_ns relative_ts = ts - ep->base_ts; // Relative to imu0 first ts + ep->playback.speed = MAX(ep->playback.speed, 1.0 / 256); + double speed = ep->playback.speed; + timepoint_ns mapped_ts = ep->start_ts + ep->offset_ts + relative_ts / speed; + return mapped_ts; +} + +static void +euroc_player_load_next_frame(struct euroc_player *ep, bool is_left, struct xrt_frame *&xf) +{ + using xrt::auxiliary::tracking::FrameMat; + img_sample sample = is_left ? ep->left_imgs->at(ep->img_seq) : ep->right_imgs->at(ep->img_seq); + ep->playback.scale = CLAMP(ep->playback.scale, 1.0 / 16, 4); + + // Load will be influenced by these playback options + bool allow_color = ep->playback.color; + float scale = ep->playback.scale; + + // Load image from disk + timepoint_ns timestamp = euroc_player_mapped_ts(ep, sample.first); + std::string img_name = sample.second; + EUROC_TRACE(ep, "%s img t = %ld filename = %s", is_left ? "left" : "right", timestamp, img_name.c_str()); + cv::ImreadModes read_mode = allow_color ? cv::IMREAD_ANYCOLOR : cv::IMREAD_GRAYSCALE; + cv::Mat img = cv::imread(img_name, read_mode); // If colored, reads in BGR order + + if (scale != 1.0) { + cv::Mat tmp; + cv::resize(img, tmp, cv::Size(), scale, scale); + img = tmp; + } + + // Create xrt_frame, it will be freed by FrameMat destructor + EUROC_ASSERT(xf == NULL || xf->reference.count > 0, "Must be given a valid or NULL frame ptr"); + EUROC_ASSERT(timestamp > 0, "Unexpected negative timestamp"); + //! @todo Not using xrt_stereo_format because we use two sinks. It would + //! probably be better to refactor everything to use stereo frames instead. + FrameMat::Params params{XRT_STEREO_FORMAT_NONE, static_cast(timestamp)}; + auto wrap = img.channels() == 3 ? FrameMat::wrapR8G8B8 : FrameMat::wrapL8; + wrap(img, &xf, params); + + // Fields that aren't set by FrameMat + xf->owner = ep; + xf->source_timestamp = sample.first; + xf->source_sequence = ep->img_seq; + xf->source_id = ep->base.source_id; +} + +static bool +euroc_player_is_imu_next(struct euroc_player *ep) +{ + bool prioritize_imus = ep->playback.send_all_imus_first; + bool more_imus = ep->imu_seq < ep->imus->size(); + if (more_imus && prioritize_imus) { + return true; + } + + bool more_imgs = ep->img_seq < ep->left_imgs->size(); + timepoint_ns imu_ts = more_imus ? ep->imus->at(ep->imu_seq).timestamp_ns : INT64_MAX; + timepoint_ns img_ts = more_imgs ? ep->left_imgs->at(ep->img_seq).first : INT64_MAX; + return imu_ts < img_ts; +} + +static void +euroc_player_push_next_sample(struct euroc_player *ep) +{ + bool stereo = ep->playback.stereo; + + // Push next IMU sample + if (euroc_player_is_imu_next(ep)) { + xrt_imu_sample sample = ep->imus->at(ep->imu_seq++); + sample.timestamp_ns = euroc_player_mapped_ts(ep, sample.timestamp_ns); + xrt_sink_push_imu(ep->in_sinks.imu, &sample); + return; + } + + // Push next frame(s) + struct xrt_frame *left_xf = NULL, *right_xf = NULL; + euroc_player_load_next_frame(ep, true, left_xf); + if (stereo) { + // TODO: Some SLAM systems expect synced frames, but that's not an + // EuRoC requirement. Adapt to work with unsynced datasets too. + euroc_player_load_next_frame(ep, false, right_xf); + EUROC_ASSERT(left_xf->timestamp == right_xf->timestamp, "Unsynced stereo frames"); + } + ep->img_seq++; + + xrt_sink_push_frame(ep->in_sinks.left, left_xf); + if (stereo) { + xrt_sink_push_frame(ep->in_sinks.right, right_xf); + } + + // We are now done with the frames, unreference them so + // they can be freed if all consumers are done with them. + xrt_frame_reference(&left_xf, NULL); + xrt_frame_reference(&right_xf, NULL); + + snprintf(ep->progress_text, sizeof(ep->progress_text), "Frames %lu/%lu - IMUs %lu/%lu", ep->img_seq, + ep->left_imgs->size(), ep->imu_seq, ep->imus->size()); + + // Determine how much to sleep until next frame + if (ep->img_seq >= ep->left_imgs->size()) { + return; + } + timepoint_ns next_frame_ts = euroc_player_mapped_ts(ep, ep->left_imgs->at(ep->img_seq).first); + timepoint_ns now = os_monotonic_get_ts(); + int32_t frame_delay_ns = MAX(next_frame_ts - now, 0); + os_nanosleep(frame_delay_ns); +} + +static bool +euroc_player_is_paused(struct euroc_player *ep) +{ + if (!ep->playback.paused) { + return false; + } + + timepoint_ns pre_pause = os_monotonic_get_ts(); + os_nanosleep(200 * 1000 * 1000); + timepoint_ns pos_pause = os_monotonic_get_ts(); + timepoint_ns pause_length = pos_pause - pre_pause; + ep->offset_ts += pause_length; + return true; +} + +static void * +euroc_player_mainloop(void *ptr) +{ + struct xrt_fs *xfs = (struct xrt_fs *)ptr; + struct euroc_player *ep = euroc_player(xfs); + EUROC_INFO(ep, "Starting euroc playback"); + + euroc_player_preload(ep); + euroc_player_user_skip(ep); + + ep->start_ts = os_monotonic_get_ts(); + + bool more_imus = ep->imu_seq < ep->imus->size(); + bool more_imgs = ep->img_seq < ep->left_imgs->size(); + + while (ep->is_running && (more_imus || more_imgs)) { + if (euroc_player_is_paused(ep)) { + continue; + } + + euroc_player_push_next_sample(ep); + + more_imus = ep->imu_seq < ep->imus->size(); + more_imgs = ep->img_seq < ep->left_imgs->size(); + } + + EUROC_INFO(ep, "Euroc dataset playback finished"); + euroc_player_set_ui_state(ep, STREAM_ENDED); + + return NULL; +} + +// Frame server functionality + +static bool +euroc_player_enumerate_modes(struct xrt_fs *xfs, struct xrt_fs_mode **out_modes, uint32_t *out_count) +{ + struct euroc_player *ep = euroc_player(xfs); + + // Should be freed by caller + struct xrt_fs_mode *modes = U_TYPED_ARRAY_CALLOC(struct xrt_fs_mode, 1); + EUROC_ASSERT(modes != NULL, "Unable to calloc euroc playback modes"); + + // At first, it would sound like a good idea to list all possible playback + // modes here, however it gets more troublesome than it is worth, and there + // doesn't seem to be a good reason to use this feature here. Having said + // that, a basic fs mode will be provided, which consists of only the original + // properties of the dataset, and ignores the other playback options that can + // be tweaked in the UI. + modes[0] = ep->mode; + + *out_modes = modes; + *out_count = 1; + + return true; +} + +static bool +euroc_player_configure_capture(struct xrt_fs *xfs, struct xrt_fs_capture_parameters *cp) +{ + // struct euroc_player *ep = euroc_player(xfs); + EUROC_ASSERT(false, "Not implemented"); + return false; +} + +static void +receive_left_frame(struct xrt_frame_sink *sink, struct xrt_frame *xf) +{ + struct euroc_player *ep = container_of(sink, struct euroc_player, left_sink); + EUROC_TRACE(ep, "left img t=%ld source_t=%ld", xf->timestamp, xf->source_timestamp); + u_sink_debug_push_frame(&ep->ui_left_sink, xf); + if (ep->out_sinks.left) { + xrt_sink_push_frame(ep->out_sinks.left, xf); + } +} + +static void +receive_right_frame(struct xrt_frame_sink *sink, struct xrt_frame *xf) +{ + struct euroc_player *ep = container_of(sink, struct euroc_player, right_sink); + EUROC_TRACE(ep, "right img t=%ld source_t=%ld", xf->timestamp, xf->source_timestamp); + u_sink_debug_push_frame(&ep->ui_right_sink, xf); + if (ep->out_sinks.right) { + xrt_sink_push_frame(ep->out_sinks.right, xf); + } +} + +static void +receive_imu_sample(struct xrt_imu_sink *sink, struct xrt_imu_sample *s) +{ + struct euroc_player *ep = container_of(sink, struct euroc_player, imu_sink); + + timepoint_ns ts = s->timestamp_ns; + xrt_vec3_f64 a = s->accel_m_s2; + xrt_vec3_f64 w = s->gyro_rad_secs; + + // UI log + const xrt_vec3 gyro{(float)w.x, (float)w.y, (float)w.z}; + const xrt_vec3 accel{(float)a.x, (float)a.y, (float)a.z}; + m_ff_vec3_f32_push(ep->gyro_ff, &gyro, ts); + m_ff_vec3_f32_push(ep->accel_ff, &accel, ts); + + // Trace log + EUROC_TRACE(ep, "imu t=%ld ax=%f ay=%f az=%f wx=%f wy=%f wz=%f", ts, a.x, a.y, a.z, w.x, w.y, w.z); + if (ep->out_sinks.imu) { + xrt_sink_push_imu(ep->out_sinks.imu, s); + } +} + +//! This is the @ref xrt_fs stream start method, however as the euroc playback +//! is heavily customizable, it will be managed through the UI. So this will not +//! really start outputting frames but mainly prepare everything to start doing +//! so when the user decides. +static bool +euroc_player_stream_start(struct xrt_fs *xfs, + struct xrt_frame_sink *xs, + enum xrt_fs_capture_type capture_type, + uint32_t descriptor_index) +{ + struct euroc_player *ep = euroc_player(xfs); + + if (xs == NULL && capture_type == XRT_FS_CAPTURE_TYPE_TRACKING) { + EUROC_INFO(ep, "Starting Euroc Player in tracking mode"); + if (ep->out_sinks.left == NULL) { + EUROC_WARN(ep, "No left sink provided, will keep running but tracking is unlikely to work"); + } + } else if (xs != NULL && capture_type == XRT_FS_CAPTURE_TYPE_CALIBRATION) { + EUROC_INFO(ep, "Starting Euroc Player in calibration mode, will stream only left frames right away"); + ep->out_sinks.left = xs; + euroc_player_start_btn_cb(ep); + } else { + EUROC_ASSERT(false, "Unsupported stream configuration xs=%p capture_type=%d", (void *)xs, capture_type); + return false; + } + + ep->is_running = true; + return ep->is_running; +} + +static bool +euroc_player_slam_stream_start(struct xrt_fs *xfs, struct xrt_slam_sinks *sinks) +{ + struct euroc_player *ep = euroc_player(xfs); + ep->out_sinks = *sinks; + return euroc_player_stream_start(xfs, NULL, XRT_FS_CAPTURE_TYPE_TRACKING, 0); +} + +static bool +euroc_player_stream_stop(struct xrt_fs *xfs) +{ + struct euroc_player *ep = euroc_player(xfs); + ep->is_running = false; + + os_thread_helper_stop(&ep->play_thread); + os_thread_helper_destroy(&ep->play_thread); + + return true; +} + +static bool +euroc_player_is_running(struct xrt_fs *xfs) +{ + struct euroc_player *ep = euroc_player(xfs); + return ep->is_running; +} + +// Frame node functionality + +static void +euroc_player_break_apart(struct xrt_frame_node *node) +{ + struct euroc_player *ep = container_of(node, struct euroc_player, node); + euroc_player_stream_stop(&ep->base); + return; +} + +static void +euroc_player_destroy(struct xrt_frame_node *node) +{ + struct euroc_player *ep = container_of(node, struct euroc_player, node); + + delete ep->imus; + delete ep->left_imgs; + delete ep->right_imgs; + + u_var_remove_root(ep); + u_sink_debug_destroy(&ep->ui_left_sink); + u_sink_debug_destroy(&ep->ui_right_sink); + m_ff_vec3_f32_free(&ep->gyro_ff); + m_ff_vec3_f32_free(&ep->accel_ff); + + free(ep); + + return; +} + +// UI functionality + +static void +euroc_player_set_ui_state(struct euroc_player *ep, euroc_player_ui_state state) +{ + // -> UNINITIALIZED -> NOT_STREAMING -> STREAM_PLAYING <-> STREAM_PAUSED + // └> STREAM_ENDED <┘ + euroc_player_ui_state prev_state = ep->ui_state; + if (state == NOT_STREAMING) { + EUROC_ASSERT_(prev_state == UNINITIALIZED); + ep->pause_btn.disabled = true; + snprintf(ep->progress_text, sizeof(ep->progress_text), "Stream has not started"); + } else if (state == STREAM_PLAYING) { + EUROC_ASSERT_(prev_state == NOT_STREAMING || prev_state == STREAM_PAUSED); + ep->start_btn.disabled = true; + ep->pause_btn.disabled = false; + snprintf(ep->pause_btn.label, sizeof(ep->pause_btn.label), "Pause"); + } else if (state == STREAM_PAUSED) { + EUROC_ASSERT_(prev_state == STREAM_PLAYING); + snprintf(ep->pause_btn.label, sizeof(ep->pause_btn.label), "Resume"); + } else if (state == STREAM_ENDED) { + EUROC_ASSERT_(prev_state == STREAM_PLAYING || prev_state == STREAM_PAUSED); + ep->pause_btn.disabled = true; + } else { + EUROC_ASSERT(false, "Unexpected UI state transition from %d to %d", prev_state, state); + } + ep->ui_state = state; +} + +static void +euroc_player_start_btn_cb(void *ptr) +{ + struct euroc_player *ep = (struct euroc_player *)ptr; + + int ret = 0; + ret |= os_thread_helper_init(&ep->play_thread); + ret |= os_thread_helper_start(&ep->play_thread, euroc_player_mainloop, ep); + EUROC_ASSERT(ret == 0, "Thread launch failure"); + + euroc_player_set_ui_state(ep, STREAM_PLAYING); +} + +static void +euroc_player_pause_btn_cb(void *ptr) +{ + struct euroc_player *ep = (struct euroc_player *)ptr; + ep->playback.paused = !ep->playback.paused; + euroc_player_set_ui_state(ep, ep->playback.paused ? STREAM_PAUSED : STREAM_PLAYING); +} + +static void +euroc_player_setup_gui(struct euroc_player *ep) +{ + // Set sinks to display in UI + u_sink_debug_init(&ep->ui_left_sink); + u_sink_debug_init(&ep->ui_right_sink); + m_ff_vec3_f32_alloc(&ep->gyro_ff, 1000); + m_ff_vec3_f32_alloc(&ep->accel_ff, 1000); + + // Set button callbacks + ep->start_btn.cb = euroc_player_start_btn_cb; + ep->start_btn.ptr = ep; + ep->pause_btn.cb = euroc_player_pause_btn_cb; + ep->pause_btn.ptr = ep; + euroc_player_set_ui_state(ep, NOT_STREAMING); + + // Add UI wigets + u_var_add_root(ep, "Euroc Player", false); + u_var_add_ro_text(ep, ep->dataset.path, "Dataset"); + u_var_add_ro_text(ep, ep->progress_text, "Progress"); + u_var_add_button(ep, &ep->start_btn, "Start"); + u_var_add_button(ep, &ep->pause_btn, "Pause"); + u_var_add_log_level(ep, &ep->ll, "Log Level"); + + u_var_add_gui_header(ep, NULL, "Playback Options"); + u_var_add_ro_text(ep, "When using a SLAM system, setting these after start is unlikely to work", "Note"); + u_var_add_bool(ep, &ep->playback.stereo, "Stereo (if available)"); + u_var_add_bool(ep, &ep->playback.color, "Color (if available)"); + u_var_add_f32(ep, &ep->playback.skip_first_s, "First seconds to skip (set at start)"); + u_var_add_f32(ep, &ep->playback.scale, "Scale"); + u_var_add_f64(ep, &ep->playback.speed, "Speed (set at start)"); + u_var_add_bool(ep, &ep->playback.send_all_imus_first, "Send all IMU samples now"); + u_var_add_bool(ep, &ep->playback.use_source_ts, "Don't correct timestamps (set at start)"); + + u_var_add_gui_header(ep, NULL, "Streams"); + u_var_add_ro_ff_vec3_f32(ep, ep->gyro_ff, "Gyroscope"); + u_var_add_ro_ff_vec3_f32(ep, ep->accel_ff, "Accelerometer"); + u_var_add_sink_debug(ep, &ep->ui_left_sink, "Left Camera"); + u_var_add_sink_debug(ep, &ep->ui_right_sink, "Right Camera"); +} + +// Euroc driver creation + +struct xrt_fs * +euroc_player_create(struct xrt_frame_context *xfctx, const char *path) +{ + struct euroc_player *ep = U_TYPED_CALLOC(struct euroc_player); + euroc_player_fill_dataset_info(ep, path); + ep->mode = xrt_fs_mode{ + ep->dataset.width, + ep->dataset.height, + ep->dataset.is_colored ? XRT_FORMAT_R8G8B8 : XRT_FORMAT_R8, + // Stereo euroc *is* supported, but we don't expose that through the + // xrt_fs interface as it will be managed through two different sinks. + XRT_STEREO_FORMAT_NONE, + }; + + // Using pointers to not mix std::vector with a C-compatible struct + ep->imus = new imu_samples{}; + ep->left_imgs = new img_samples{}; + ep->right_imgs = new img_samples{}; + + ep->playback.stereo = ep->dataset.is_stereo; + ep->playback.color = ep->dataset.is_colored; + ep->playback.skip_first_s = 0.0; + ep->playback.scale = 1.0; + ep->playback.speed = 1.0; + ep->playback.send_all_imus_first = false; + ep->playback.use_source_ts = false; + + ep->ll = debug_get_log_option_euroc_log(); + euroc_player_setup_gui(ep); + + ep->left_sink.push_frame = receive_left_frame; + ep->right_sink.push_frame = receive_right_frame; + ep->imu_sink.push_imu = receive_imu_sample; + ep->in_sinks.left = &ep->left_sink; + ep->in_sinks.right = &ep->right_sink; + ep->in_sinks.imu = &ep->imu_sink; + ep->out_sinks = {0, 0, 0}; + + struct xrt_fs *xfs = &ep->base; + xfs->enumerate_modes = euroc_player_enumerate_modes; + xfs->configure_capture = euroc_player_configure_capture; + xfs->stream_start = euroc_player_stream_start; + xfs->slam_stream_start = euroc_player_slam_stream_start; + xfs->stream_stop = euroc_player_stream_stop; + xfs->is_running = euroc_player_is_running; + + snprintf(xfs->name, sizeof(xfs->name), EUROC_PLAYER_STR); + snprintf(xfs->product, sizeof(xfs->product), EUROC_PLAYER_STR " Product"); + snprintf(xfs->manufacturer, sizeof(xfs->manufacturer), EUROC_PLAYER_STR " Manufacturer"); + snprintf(xfs->serial, sizeof(xfs->serial), EUROC_PLAYER_STR " Serial"); + xfs->source_id = 0xECD0FEED; + + struct xrt_frame_node *xfn = &ep->node; + xfn->break_apart = euroc_player_break_apart; + xfn->destroy = euroc_player_destroy; + + xrt_frame_context_add(xfctx, &ep->node); + + EUROC_DEBUG(ep, "Euroc player created"); + + return xfs; +} diff --git a/src/xrt/drivers/hdk/hdk_device.cpp b/src/xrt/drivers/hdk/hdk_device.cpp index 90f3e6efb..d2ca2a2d7 100644 --- a/src/xrt/drivers/hdk/hdk_device.cpp +++ b/src/xrt/drivers/hdk/hdk_device.cpp @@ -99,6 +99,9 @@ hdk_device_destroy(struct xrt_device *xdev) os_thread_helper_destroy(&hd->imu_thread); + // Now that the thread is not running we can destroy the lock. + os_mutex_destroy(&hd->lock); + if (hd->dev != NULL) { os_hid_destroy(hd->dev); hd->dev = NULL; @@ -204,9 +207,12 @@ hdk_device_update(struct hdk_device *hd) math_quat_rotate(&ang_vel_quat, &rot_90_about_x, &ang_vel_quat); math_quat_rotate(&negative_90_about_x, &ang_vel_quat, &ang_vel_quat); + os_mutex_lock(&hd->lock); hd->ang_vel_quat = ang_vel_quat; hd->quat_valid = true; + os_mutex_unlock(&hd->lock); + return 1; } @@ -228,55 +234,38 @@ hdk_device_get_tracked_pose(struct xrt_device *xdev, // Adjusting for latency - 14ms, found empirically. now -= 14000000; + os_mutex_lock(&hd->lock); if (!hd->quat_valid) { out_relation->relation_flags = XRT_SPACE_RELATION_BITMASK_NONE; HDK_TRACE(hd, "GET_TRACKED_POSE: No pose"); + os_mutex_unlock(&hd->lock); return; } - os_thread_helper_lock(&hd->imu_thread); - out_relation->pose.orientation = hd->quat; out_relation->angular_velocity.x = hd->ang_vel_quat.x; out_relation->angular_velocity.y = hd->ang_vel_quat.y; out_relation->angular_velocity.z = hd->ang_vel_quat.z; + os_mutex_unlock(&hd->lock); + out_relation->relation_flags = xrt_space_relation_flags(XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ANGULAR_VELOCITY_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT); - os_thread_helper_unlock(&hd->imu_thread); - HDK_TRACE(hd, "GET_TRACKED_POSE (%f, %f, %f, %f) ANG_VEL (%f, %f, %f)", hd->quat.x, hd->quat.y, hd->quat.z, hd->quat.w, hd->ang_vel_quat.x, hd->ang_vel_quat.y, hd->ang_vel_quat.z); } static void hdk_device_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { - struct xrt_pose pose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; - bool adjust = view_index == 0; - - pose.position.x = eye_relation->x / 2.0f; - pose.position.y = eye_relation->y / 2.0f; - pose.position.z = eye_relation->z / 2.0f; - - // Adjust for left/right while also making sure there aren't any -0.f. - if (pose.position.x > 0.0f && adjust) { - pose.position.x = -pose.position.x; - } - if (pose.position.y > 0.0f && adjust) { - pose.position.y = -pose.position.y; - } - if (pose.position.z > 0.0f && adjust) { - pose.position.z = -pose.position.z; - } - - *out_pose = pose; + (void)xdev; + u_device_get_view_pose(eye_relation, view_index, out_pose); } static void * @@ -315,7 +304,10 @@ hdk_device_create(struct os_hid_device *dev, enum HDK_VARIANT variant) (enum u_device_alloc_flags)(U_DEVICE_ALLOC_HMD | U_DEVICE_ALLOC_TRACKING_NONE); struct hdk_device *hd = U_DEVICE_ALLOCATE(struct hdk_device, flags, 1, 0); - hd->base.hmd->blend_mode = XRT_BLEND_MODE_OPAQUE; + size_t idx = 0; + hd->base.hmd->blend_modes[idx++] = XRT_BLEND_MODE_OPAQUE; + hd->base.hmd->num_blend_modes = idx; + hd->base.update_inputs = hdk_device_update_inputs; hd->base.get_tracked_pose = hdk_device_get_tracked_pose; hd->base.get_view_pose = hdk_device_get_view_pose; @@ -326,6 +318,7 @@ hdk_device_create(struct os_hid_device *dev, enum HDK_VARIANT variant) hd->ll = debug_get_log_option_hdk_log(); snprintf(hd->base.str, XRT_DEVICE_NAME_LEN, "OSVR HDK-family Device"); + snprintf(hd->base.serial, XRT_DEVICE_NAME_LEN, "OSVR HDK-family Device"); if (variant == HDK_UNKNOWN) { HDK_ERROR(hd, "Don't know which HDK variant this is."); @@ -486,7 +479,15 @@ hdk_device_create(struct os_hid_device *dev, enum HDK_VARIANT variant) // } if (hd->dev) { - int ret = os_thread_helper_start(&hd->imu_thread, hdk_device_run_thread, hd); + // Mutex before thread. + int ret = os_mutex_init(&hd->lock); + if (ret != 0) { + HDK_ERROR(hd, "Failed to init mutex!"); + hdk_device_destroy(&hd->base); + return NULL; + } + + ret = os_thread_helper_start(&hd->imu_thread, hdk_device_run_thread, hd); if (ret != 0) { HDK_ERROR(hd, "Failed to start mainboard thread!"); hdk_device_destroy((struct xrt_device *)hd); diff --git a/src/xrt/drivers/hdk/hdk_device.h b/src/xrt/drivers/hdk/hdk_device.h index 39ff10a55..8956d3463 100644 --- a/src/xrt/drivers/hdk/hdk_device.h +++ b/src/xrt/drivers/hdk/hdk_device.h @@ -35,6 +35,7 @@ struct hdk_device enum HDK_VARIANT variant; struct os_thread_helper imu_thread; + struct os_mutex lock; enum u_logging_level ll; bool disconnect_notified; diff --git a/src/xrt/drivers/ht/ht_algorithm.hpp b/src/xrt/drivers/ht/ht_algorithm.hpp new file mode 100644 index 000000000..211bc12c2 --- /dev/null +++ b/src/xrt/drivers/ht/ht_algorithm.hpp @@ -0,0 +1,775 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Camera based hand tracking mainloop algorithm. + * @author Moses Turner + * @ingroup drv_ht + */ + +#pragma once + +#include "cjson/cJSON.h" +#include "math/m_filter_one_euro.h" +#include "os/os_time.h" +#include "util/u_frame.h" + +#include "templates/NaivePermutationSort.hpp" + +#include "ht_driver.hpp" +#include "ht_models.hpp" +#include "ht_hand_math.hpp" +#include "ht_image_math.hpp" +#include "util/u_time.h" + +#include +#include + + +// Flags to tell state tracker that these are indeed valid joints +static enum xrt_space_relation_flags valid_flags_ht = (enum xrt_space_relation_flags)( + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | + XRT_SPACE_RELATION_POSITION_VALID_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT); + + +static void +htProcessJoint(struct ht_device *htd, + struct xrt_vec3 model_out, + struct xrt_hand_joint_set *hand, + enum xrt_hand_joint idx) +{ + hand->values.hand_joint_set_default[idx].relation.relation_flags = valid_flags_ht; + hand->values.hand_joint_set_default[idx].relation.pose.position.x = model_out.x; + hand->values.hand_joint_set_default[idx].relation.pose.position.y = model_out.y; + hand->values.hand_joint_set_default[idx].relation.pose.position.z = model_out.z; +} + +static float +errHistory2D(HandHistory2DBBox *past, Palm7KP *present) +{ + if (!past->htAlgorithm_approves) { + // U_LOG_E("Returning big number because htAlgorithm told me to!"); + return 100000000000000000000000000000.0f; + } + float sum_of_lengths = m_vec2_len(*past->wrist_unfiltered[0] - *past->middle_unfiltered[0]) + + m_vec2_len(present->kps[WRIST_7KP] - present->kps[MIDDLE_7KP]); + + float sum_of_distances = (m_vec2_len(*past->wrist_unfiltered[0] - present->kps[WRIST_7KP]) + + m_vec2_len(*past->middle_unfiltered[0] - present->kps[MIDDLE_7KP])); + + + float final = sum_of_distances / sum_of_lengths; + + return final; +} + +static std::vector +htImageToKeypoints(struct ht_view *htv) +{ + int view = htv->view; + struct ht_device *htd = htv->htd; + + + cv::Mat raw_input = htv->run_model_on_this; + + // Get a list of palms - drop confidences and ssd bounding boxes, just keypoints. + + std::vector hand_detections = htv->run_detection_model(htv, raw_input); + + std::vector used_histories; + std::vector used_detections; + + std::vector history_indices; + std::vector detection_indices; + std::vector dontuse; + + + // Strategy here is: We have a big list of palms. Match 'em up to previous palms. + naive_sort_permutation_by_error(htv->bbox_histories, hand_detections, + + // bools + used_histories, used_detections, + + history_indices, detection_indices, dontuse, + errHistory2D, 1.0f); + + // Here's the trick - we use the associated bbox_filter to get an output but *never commit* the noisy 128x128 + // detection; instead later on we commit the (hopefully) nicer palm and wrist from the 224x224 keypoint + // estimation. + + // Add extra detections! + for (size_t i = 0; i < used_detections.size(); i++) { + if ((used_detections[i] == false) && hand_detections[i].confidence > 0.65) { + // Confidence to get in the door is 0.65, confidence to stay in is 0.3 + HandHistory2DBBox hist_new = {}; + m_filter_euro_vec2_init(&hist_new.m_filter_center, FCMIN_BBOX_POSITION, FCMIN_D_BB0X_POSITION, + BETA_BB0X_POSITION); + m_filter_euro_vec2_init(&hist_new.m_filter_direction, FCMIN_BBOX_ORIENTATION, + FCMIN_D_BB0X_ORIENTATION, BETA_BB0X_ORIENTATION); + + htv->bbox_histories.push_back(hist_new); + history_indices.push_back(htv->bbox_histories.size() - 1); + detection_indices.push_back(i); + } + } + + // Do the things for each active bbox history! + for (size_t i = 0; i < history_indices.size(); i++) { + HandHistory2DBBox *hist_of_interest = &htv->bbox_histories[history_indices[i]]; + hist_of_interest->wrist_unfiltered.push(hand_detections[detection_indices[i]].kps[WRIST_7KP]); + hist_of_interest->index_unfiltered.push(hand_detections[detection_indices[i]].kps[INDEX_7KP]); + hist_of_interest->middle_unfiltered.push(hand_detections[detection_indices[i]].kps[MIDDLE_7KP]); + hist_of_interest->pinky_unfiltered.push(hand_detections[detection_indices[i]].kps[LITTLE_7KP]); + // Eh do the rest later + } + + // Prune stale detections! (After we don't need {history,detection}_indices to be correct) + int bob = 0; + for (size_t i = 0; i < used_histories.size(); i++) { + if (used_histories[i] == false) { + // history never got assigned a present hand to it. treat it as stale delete it. + + HT_TRACE(htv->htd, "Removing bbox from history!\n"); + htv->bbox_histories.erase(htv->bbox_histories.begin() + i + bob); + bob--; + } + } + if (htv->bbox_histories.size() == 0) { + return {}; // bail early + } + + + + std::vector list_of_hands_in_bbox( + htv->bbox_histories.size()); // all of these are same size as htv->bbox_histories + + std::vector> await_list_of_hand_in_bbox; //(htv->bbox_histories.size()); + + std::vector blah(htv->bbox_histories.size()); + + std::vector output; + + if (htv->bbox_histories.size() > 2) { + HT_DEBUG(htd, "More than two hands (%zu) in 2D view %i", htv->bbox_histories.size(), htv->view); + } + + + for (size_t i = 0; i < htv->bbox_histories.size(); i++) { //(BBoxHistory * entry : htv->bbox_histories) { + HandHistory2DBBox *entry = &htv->bbox_histories[i]; + cv::Mat hand_rect = cv::Mat(224, 224, CV_8UC3); + xrt_vec2 unfiltered_middle; + xrt_vec2 unfiltered_direction; + + + centerAndRotationFromJoints(htv, entry->wrist_unfiltered[0], entry->index_unfiltered[0], + entry->middle_unfiltered[0], entry->pinky_unfiltered[0], &unfiltered_middle, + &unfiltered_direction); + + xrt_vec2 filtered_middle; + xrt_vec2 filtered_direction; + + m_filter_euro_vec2_run_no_commit(&entry->m_filter_center, htv->htd->current_frame_timestamp, + &unfiltered_middle, &filtered_middle); + m_filter_euro_vec2_run_no_commit(&entry->m_filter_direction, htv->htd->current_frame_timestamp, + &unfiltered_direction, &filtered_direction); + + rotatedRectFromJoints(htv, filtered_middle, filtered_direction, &blah[i]); + + warpAffine(raw_input, hand_rect, blah[i].warp_there, hand_rect.size()); + + await_list_of_hand_in_bbox.push_back( + std::async(std::launch::async, htd->views[view].run_keypoint_model, &htd->views[view], hand_rect)); + } + + // cut here + + for (size_t i = 0; i < htv->bbox_histories.size(); i++) { + + Hand2D in_bbox = await_list_of_hand_in_bbox[i].get(); + + cv::Matx23f warp_back = blah[i].warp_back; + + Hand2D in_image_ray_coords; + Hand2D in_image_px_coords; + + for (int i = 0; i < 21; i++) { + struct xrt_vec3 vec = in_bbox.kps[i]; + +#if 1 + xrt_vec3 rr = transformVecBy2x3(vec, warp_back); + rr.z = vec.z; +#else + xrt_vec3 rr; + rr.x = (vec.x * warp_back(0, 0)) + (vec.y * warp_back(0, 1)) + warp_back(0, 2); + rr.y = (vec.x * warp_back(1, 0)) + (vec.y * warp_back(1, 1)) + warp_back(1, 2); + rr.z = vec.z; +#endif + in_image_px_coords.kps[i] = rr; + + in_image_ray_coords.kps[i] = raycoord(htv, rr); + if (htd->debug_scribble && htd->dynamic_config.scribble_2d_keypoints) { + handDot(htv->debug_out_to_this, {rr.x, rr.y}, fmax((-vec.z + 100 - 20) * .08, 2), + ((float)i) / 21.0f, 0.95f, cv::FILLED); + } + } + xrt_vec2 wrist_in_px_coords = {in_image_px_coords.kps[WRIST].x, in_image_px_coords.kps[WRIST].y}; + xrt_vec2 index_in_px_coords = {in_image_px_coords.kps[INDX_PXM].x, in_image_px_coords.kps[INDX_PXM].y}; + xrt_vec2 middle_in_px_coords = {in_image_px_coords.kps[MIDL_PXM].x, in_image_px_coords.kps[MIDL_PXM].y}; + xrt_vec2 little_in_px_coords = {in_image_px_coords.kps[LITL_PXM].x, in_image_px_coords.kps[LITL_PXM].y}; + xrt_vec2 dontuse; + + xrt_vec2 unfiltered_middle, unfiltered_direction; + + centerAndRotationFromJoints(htv, &wrist_in_px_coords, &index_in_px_coords, &middle_in_px_coords, + &little_in_px_coords, &unfiltered_middle, &unfiltered_direction); + + m_filter_euro_vec2_run(&htv->bbox_histories[i].m_filter_center, htv->htd->current_frame_timestamp, + &unfiltered_middle, &dontuse); + + m_filter_euro_vec2_run(&htv->bbox_histories[i].m_filter_direction, htv->htd->current_frame_timestamp, + &unfiltered_direction, &dontuse); + + output.push_back(in_image_ray_coords); + } + return output; +} + +#if defined(EXPERIMENTAL_DATASET_RECORDING) + +static void +jsonAddJoint(cJSON *into_this, xrt_pose loc, const char *name) +{ + + cJSON *container = cJSON_CreateObject(); + cJSON *joint_loc = cJSON_CreateArray(); + cJSON_AddItemToArray(joint_loc, cJSON_CreateNumber(loc.position.x)); + cJSON_AddItemToArray(joint_loc, cJSON_CreateNumber(loc.position.y)); + cJSON_AddItemToArray(joint_loc, cJSON_CreateNumber(loc.position.z)); + + cJSON_AddItemToObject(container, "position", joint_loc); + + cJSON *joint_rot = cJSON_CreateArray(); + + + cJSON_AddItemToArray(joint_rot, cJSON_CreateNumber(loc.orientation.x)); + cJSON_AddItemToArray(joint_rot, cJSON_CreateNumber(loc.orientation.y)); + cJSON_AddItemToArray(joint_rot, cJSON_CreateNumber(loc.orientation.z)); + cJSON_AddItemToArray(joint_rot, cJSON_CreateNumber(loc.orientation.w)); + + cJSON_AddItemToObject(container, "rotation_quat_xyzw", joint_rot); + + cJSON_AddItemToObject(into_this, name, container); +} + +void +jsonMaybeAddSomeHands(struct ht_device *htd, bool err) +{ + if (!htd->tracking_should_record_dataset) { + return; + } + cJSON *j_this_frame = cJSON_CreateObject(); + cJSON_AddItemToObject(j_this_frame, "seq_since_start", cJSON_CreateNumber(htd->gst.current_index)); + cJSON_AddItemToObject(j_this_frame, "seq_src", cJSON_CreateNumber(htd->frame_for_process->source_sequence)); + cJSON_AddItemToObject(j_this_frame, "ts", cJSON_CreateNumber(htd->gst.last_frame_ns)); + + cJSON *j_hands_in_frame = cJSON_AddArrayToObject(j_this_frame, "detected_hands"); + if (!err) { + for (size_t idx_hand = 0; idx_hand < htd->histories_3d.size(); idx_hand++) { + cJSON *j_hand_in_frame = cJSON_CreateObject(); + + cJSON *j_uuid = cJSON_CreateNumber(htd->histories_3d[idx_hand].uuid); + cJSON_AddItemToObject(j_hand_in_frame, "uuid", j_uuid); + + cJSON *j_handedness = cJSON_CreateNumber(htd->histories_3d[idx_hand].handedness); + cJSON_AddItemToObject(j_hand_in_frame, "handedness", j_handedness); + + static const char *keys[21] = { + "WRIST", + + "THMB_MCP", "THMB_PXM", "THMB_DST", "THMB_TIP", + + "INDX_PXM", "INDX_INT", "INDX_DST", "INDX_TIP", + + "MIDL_PXM", "MIDL_INT", "MIDL_DST", "MIDL_TIP", + + "RING_PXM", "RING_INT", "RING_DST", "RING_TIP", + + "LITL_PXM", "LITL_INT", "LITL_DST", "LITL_TIP", + }; + + for (int idx_joint = 0; idx_joint < 21; idx_joint++) { + // const char* key = keys[idx_joint]; + cJSON *j_vec3 = cJSON_AddArrayToObject(j_hand_in_frame, keys[idx_joint]); + cJSON_AddItemToArray( + j_vec3, + cJSON_CreateNumber( + htd->histories_3d[idx_hand].last_hands_unfiltered[0]->kps[idx_joint].x)); + cJSON_AddItemToArray( + j_vec3, + cJSON_CreateNumber( + htd->histories_3d[idx_hand].last_hands_unfiltered[0]->kps[idx_joint].y)); + cJSON_AddItemToArray( + j_vec3, + cJSON_CreateNumber( + htd->histories_3d[idx_hand].last_hands_unfiltered[0]->kps[idx_joint].z)); + } + + + cJSON_AddItemToArray(j_hands_in_frame, j_hand_in_frame); + } + } + cJSON_AddItemToArray(htd->output_array, j_this_frame); +} + +#endif + + + +static void +htExitFrame(struct ht_device *htd, + bool err, + struct xrt_hand_joint_set final_hands_ordered_by_handedness[2], + uint64_t timestamp) +{ + + os_mutex_lock(&htd->openxr_hand_data_mediator); + if (err) { + htd->hands_for_openxr[0].is_active = false; + htd->hands_for_openxr[1].is_active = false; + } else { + memcpy(&htd->hands_for_openxr[0], &final_hands_ordered_by_handedness[0], + sizeof(struct xrt_hand_joint_set)); + memcpy(&htd->hands_for_openxr[1], &final_hands_ordered_by_handedness[1], + sizeof(struct xrt_hand_joint_set)); + htd->hands_for_openxr_timestamp = timestamp; + HT_DEBUG(htd, "Adding ts %zu", htd->hands_for_openxr_timestamp); + } + os_mutex_unlock(&htd->openxr_hand_data_mediator); +#ifdef EXPERIMENTAL_DATASET_RECORDING + if (htd->tracking_should_record_dataset) { + // Add nothing-entry to json file. + jsonMaybeAddSomeHands(htd, err); + htd->gst.current_index++; + } +#endif +} + + +static void +htJointDisparityMath(struct ht_device *htd, Hand2D *hand_in_left, Hand2D *hand_in_right, Hand3D *out_hand) +{ + for (int i = 0; i < 21; i++) { + // Believe it or not, this is where the 3D stuff happens! + float t = htd->baseline / (hand_in_left->kps[i].x - hand_in_right->kps[i].x); + + out_hand->kps[i].z = -t; + + out_hand->kps[i].x = (hand_in_left->kps[i].x * t); + out_hand->kps[i].y = -hand_in_left->kps[i].y * t; + + out_hand->kps[i].x += htd->baseline + (hand_in_right->kps[i].x * t); + out_hand->kps[i].y += -hand_in_right->kps[i].y * t; + + out_hand->kps[i].x *= .5; + out_hand->kps[i].y *= .5; + } +} +int64_t last_frame, this_frame; + +static void +htRunAlgorithm(struct ht_device *htd) +{ + XRT_TRACE_MARKER(); + +#ifdef EXPERIMENTAL_DATASET_RECORDING + + if (htd->tracking_should_record_dataset) { + U_LOG_E("PUSHING!"); + uint64_t start = os_monotonic_get_ns(); + xrt_sink_push_frame(htd->gst.sink, htd->frame_for_process); + uint64_t end = os_monotonic_get_ns(); + + if ((end - start) > 0.1 * U_TIME_1MS_IN_NS) { + U_LOG_E("Encoder overloaded!"); + } + + htd->gst.offset_ns = gstreamer_sink_get_timestamp_offset(htd->gst.gs); + htd->gst.last_frame_ns = htd->frame_for_process->timestamp - htd->gst.offset_ns; + } +#endif + + htd->current_frame_timestamp = htd->frame_for_process->timestamp; + + int64_t start, end; + start = os_monotonic_get_ns(); + + + /* + * Setup views. + */ + + const int full_width = htd->frame_for_process->width; + const int full_height = htd->frame_for_process->height; + const int view_width = htd->camera.one_view_size_px.w; + const int view_height = htd->camera.one_view_size_px.h; + + // assert(full_width == view_width * 2); + assert(full_height == view_height); + + const cv::Size full_size = cv::Size(full_width, full_height); + const cv::Size view_size = cv::Size(view_width, view_height); + const cv::Point view_offsets[2] = {cv::Point(0, 0), cv::Point(view_width, 0)}; + + cv::Mat full_frame(full_size, CV_8UC3, htd->frame_for_process->data, htd->frame_for_process->stride); + htd->views[0].run_model_on_this = full_frame(cv::Rect(view_offsets[0], view_size)); + htd->views[1].run_model_on_this = full_frame(cv::Rect(view_offsets[1], view_size)); + + htd->mat_for_process = &full_frame; + + // Check this every frame. We really, really, really don't want it to ever suddenly be null. + htd->debug_scribble = htd->debug_sink.sink != nullptr; + + cv::Mat debug_output = {}; + xrt_frame *debug_frame = nullptr; // only use if htd->debug_scribble + + if (htd->debug_scribble) { + u_frame_clone(htd->frame_for_process, &debug_frame); + debug_output = cv::Mat(full_size, CV_8UC3, debug_frame->data, debug_frame->stride); + htd->views[0].debug_out_to_this = debug_output(cv::Rect(view_offsets[0], view_size)); + htd->views[1].debug_out_to_this = debug_output(cv::Rect(view_offsets[1], view_size)); + } + + + /* + * Do the hand tracking! + */ + + std::future> future_left = + std::async(std::launch::async, htImageToKeypoints, &htd->views[0]); + std::future> future_right = + std::async(std::launch::async, htImageToKeypoints, &htd->views[1]); + std::vector hands_in_left_view = future_left.get(); + std::vector hands_in_right_view = future_right.get(); + + end = os_monotonic_get_ns(); + + + this_frame = os_monotonic_get_ns(); + + double time_ms = (double)(end - start) / (double)U_TIME_1MS_IN_NS; + double _1_time = 1 / (time_ms * 0.001); + + char t[64]; + char t2[64]; + sprintf(t, "% 8.2f ms", time_ms); + sprintf(t2, "% 8.2f fps", _1_time); + last_frame = this_frame; + + + if (htd->debug_scribble) { + cv::putText(debug_output, t, cv::Point(30, 60), cv::FONT_HERSHEY_SIMPLEX, 1.0f, cv::Scalar(0, 255, 0), + 4); + cv::putText(debug_output, t2, cv::Point(30, 100), cv::FONT_HERSHEY_SIMPLEX, 1.0f, cv::Scalar(0, 255, 0), + 4); + } else { + HT_DEBUG(htd, "%s", t); + HT_DEBUG(htd, "%s", t2); + } + + + // Convenience + uint64_t timestamp = htd->frame_for_process->timestamp; + + if (htd->debug_scribble) { + u_sink_debug_push_frame(&htd->debug_sink, debug_frame); + xrt_frame_reference(&debug_frame, NULL); + } + + // Bail early this frame if no hands were detected. + // In the long run, this'll be a silly thing - we shouldn't always take the detection model's word for it + // especially when part of the pipeline is an arbitrary confidence threshold. + if (hands_in_left_view.size() == 0 || hands_in_right_view.size() == 0) { + htExitFrame(htd, true, NULL, 0); + return; + } + + + + std::vector possible_3d_hands; + + // for every possible combination of hands in left view and hands in right view, + for (size_t idx_l = 0; idx_l < hands_in_left_view.size(); idx_l++) { + for (size_t idx_r = 0; idx_r < hands_in_right_view.size(); idx_r++) { + Hand3D cur_hand = {}; + + Hand2D &left_2d = hands_in_left_view[idx_l]; + Hand2D &right_2d = hands_in_right_view[idx_r]; + + // Calculate a 3D hand for this combination + htJointDisparityMath(htd, &hands_in_left_view[idx_l], &hands_in_right_view[idx_r], &cur_hand); + cur_hand.timestamp = timestamp; + cur_hand.rejected_by_smush = false; + + cur_hand.idx_l = idx_l; + cur_hand.idx_r = idx_r; + + // Calculate a y-disparity for this combination + cur_hand.y_disparity_error = errHandDisparity(&left_2d, &right_2d); + + possible_3d_hands.push_back(cur_hand); + } + } + + HT_DEBUG(htd, "Starting with %zu hands!", possible_3d_hands.size()); + + // For each pair of 3D hands we just made + for (size_t idx_one = 0; idx_one < possible_3d_hands.size(); idx_one++) { + for (size_t idx_two = 0; idx_two < possible_3d_hands.size(); idx_two++) { + if ((idx_one <= idx_two)) { + continue; + } + + // See if this pair is suspiciously close together. + // If it is, then this pairing is wrong - this is what was causing the "hands smushing together" + // issue - we weren't catching these reliably. + float errr = sumOfHandJointDistances(&possible_3d_hands[idx_one], &possible_3d_hands[idx_two]); + HT_TRACE(htd, "%zu %zu is smush %f", idx_one, idx_two, errr); + if (errr < 0.03f * 21.0f) { + possible_3d_hands[idx_one].rejected_by_smush = true; + possible_3d_hands[idx_two].rejected_by_smush = true; + } + } + } + + std::vector hands_unfiltered; + + for (Hand3D hand : possible_3d_hands) { + // If none of these are false, then all our heuristics indicate this is a real hand, so we add it to our + // list of real hands. + bool selected = !hand.rejected_by_smush && // + hand.y_disparity_error < 1.0f && // + rejectTooClose(htd, &hand) && // + rejectTooFar(htd, &hand) && // + rejectTinyPalm(htd, &hand); + if (selected) { + HT_TRACE(htd, "Pushing back with y-error %f", hand.y_disparity_error); + hands_unfiltered.push_back(hand); + } + } + + + std::vector past_hands_taken; + std::vector present_hands_taken; + + std::vector past_indices; + std::vector present_indices; + std::vector flow_errors; + + + float max_dist_between_frames = 1.0f; + + naive_sort_permutation_by_error(htd->histories_3d, // past + hands_unfiltered, // present + + + // outputs + past_hands_taken, present_hands_taken, past_indices, + present_indices, flow_errors, errHandHistory, + (max_dist_between_frames * 21.0f) + + ); + + + for (size_t i = 0; i < past_indices.size(); i++) { + htd->histories_3d[past_indices[i]].last_hands_unfiltered.push(hands_unfiltered[present_indices[i]]); + } + // The above may not do anything, because we'll start out with no hand histories! All the numbers of elements + // should be zero. + + + for (size_t i = 0; i < present_hands_taken.size(); i++) { + if (present_hands_taken[i] == false) { + // if this hand never got assigned to a history + HandHistory3D history_new; + history_new.uuid = rand(); // Not a great uuid, huh? Good enough for us, this only has to be + // unique across say an hour period max. + handEuroFiltersInit(&history_new, FCMIN_HAND, FCMIN_D_HAND, BETA_HAND); + history_new.last_hands_unfiltered.push(hands_unfiltered[i]); + // history_new. + htd->histories_3d.push_back( + history_new); // Add something to the end - don't initialize any of it. + } + } + + int bob = 0; + for (size_t i = 0; i < past_hands_taken.size(); i++) { + if (past_hands_taken[i] == false) { + htd->histories_3d.erase(htd->histories_3d.begin() + i + bob); + bob--; + } + } + + if (htd->histories_3d.size() == 0) { + HT_DEBUG(htd, "Bailing"); + htExitFrame(htd, true, NULL, 0); + return; + } + + size_t num_hands = htd->histories_3d.size(); + // if (num_hands > 2) { + HT_DEBUG(htd, "Ending with %zu hands!", + num_hands); // this is quite bad, but rarely happens. + // } + + // Here, we go back to our bbox_histories and remove the histories for any bounding boxes that never turned into + // good hands. + + // Iterate over all hands we're keeping track of, compute their current handedness. + std::vector valid_2d_idxs[2]; + + + for (size_t i = 0; i < htd->histories_3d.size(); i++) { + // U_LOG_E("Valid hand %zu l_idx %i r_idx %i", i, htd->histories_3d[i].last_hands[0]->idx_l, + // htd->histories_3d[i].last_hands[0]->idx_r); + valid_2d_idxs[0].push_back(htd->histories_3d[i].last_hands_unfiltered[0]->idx_l); + valid_2d_idxs[1].push_back(htd->histories_3d[i].last_hands_unfiltered[0]->idx_r); + handednessHandHistory3D(&htd->histories_3d[i]); + } + + // Almost certainly not the cleanest way of doing this but leave me alone + // Per camera view + for (int view = 0; view < 2; view++) { + // Per entry in bbox_histories + for (size_t hist_idx = 0; hist_idx < htd->views[view].bbox_histories.size(); hist_idx++) { + // See if this entry in bbox_histories ever turned into a 3D hand. If not, we notify (in a very + // silly way) htImageToKeypoints that it should go away because it was an erroneous detection. + for (size_t valid_idx : valid_2d_idxs[view]) { + if (valid_idx == hist_idx) { + htd->views[view].bbox_histories[hist_idx].htAlgorithm_approves = true; + break; + } else { + htd->views[view].bbox_histories[hist_idx].htAlgorithm_approves = false; + } + } + } + } + + // Whoo! Okay, now we have some unfiltered hands in htd->histories_3d[i].last_hands[0]! Euro filter them! + + std::vector filtered_hands(num_hands); + + for (size_t hand_index = 0; hand_index < num_hands; hand_index++) { + handEuroFiltersRun(htd, &htd->histories_3d[hand_index], &filtered_hands[hand_index]); + htd->histories_3d[hand_index].last_hands_filtered.push(filtered_hands[hand_index]); + applyThumbIndexDrag(&filtered_hands[hand_index]); + filtered_hands[hand_index].handedness = htd->histories_3d[hand_index].handedness; + } + + std::vector xr_indices; + std::vector hands_to_use; + + if (filtered_hands.size() == 1) { + if (filtered_hands[0].handedness < 0) { + // Left + xr_indices = {0}; + hands_to_use = {&filtered_hands[0]}; + } else { + xr_indices = {1}; + hands_to_use = {&filtered_hands[0]}; + } + } else { + // filtered_hands better be two for now. + if (filtered_hands[0].handedness < filtered_hands[1].handedness) { + xr_indices = {0, 1}; + hands_to_use = {&filtered_hands[0], &filtered_hands[1]}; + } else { + xr_indices = {1, 0}; + hands_to_use = {&filtered_hands[0], &filtered_hands[1]}; + } + } + + struct xrt_hand_joint_set final_hands_ordered_by_handedness[2]; + memset(&final_hands_ordered_by_handedness[0], 0, sizeof(xrt_hand_joint_set)); + memset(&final_hands_ordered_by_handedness[1], 0, sizeof(xrt_hand_joint_set)); + final_hands_ordered_by_handedness[0].is_active = false; + final_hands_ordered_by_handedness[1].is_active = false; + + for (size_t i = 0; (i < xr_indices.size()); i++) { + Hand3D *hand = hands_to_use[i]; + + struct xrt_hand_joint_set *put_in_set = &final_hands_ordered_by_handedness[xr_indices[i]]; + + xrt_vec3 wrist = hand->kps[0]; + + xrt_vec3 index_prox = hand->kps[5]; + xrt_vec3 middle_prox = hand->kps[9]; + xrt_vec3 ring_prox = hand->kps[13]; + xrt_vec3 pinky_prox = hand->kps[17]; + + xrt_vec3 middle_to_index = m_vec3_sub(index_prox, middle_prox); + xrt_vec3 middle_to_ring = m_vec3_sub(ring_prox, middle_prox); + xrt_vec3 middle_to_pinky = m_vec3_sub(pinky_prox, middle_prox); + + xrt_vec3 three_fourths_down_middle_mcp = + m_vec3_add(m_vec3_mul_scalar(wrist, 3.0f / 4.0f), m_vec3_mul_scalar(middle_prox, 1.0f / 4.0f)); + + xrt_vec3 middle_metacarpal = three_fourths_down_middle_mcp; + + float s = 0.6f; + + xrt_vec3 index_metacarpal = middle_metacarpal + m_vec3_mul_scalar(middle_to_index, s); + xrt_vec3 ring_metacarpal = middle_metacarpal + m_vec3_mul_scalar(middle_to_ring, s); + xrt_vec3 pinky_metacarpal = middle_metacarpal + m_vec3_mul_scalar(middle_to_pinky, s); + + float palm_ness = 0.33; + xrt_vec3 palm = + m_vec3_add(m_vec3_mul_scalar(wrist, palm_ness), m_vec3_mul_scalar(middle_prox, (1.0f - palm_ness))); + + + + htProcessJoint(htd, palm, put_in_set, XRT_HAND_JOINT_PALM); + + htProcessJoint(htd, hand->kps[0], put_in_set, XRT_HAND_JOINT_WRIST); + htProcessJoint(htd, hand->kps[1], put_in_set, XRT_HAND_JOINT_THUMB_METACARPAL); + htProcessJoint(htd, hand->kps[2], put_in_set, XRT_HAND_JOINT_THUMB_PROXIMAL); + htProcessJoint(htd, hand->kps[3], put_in_set, XRT_HAND_JOINT_THUMB_DISTAL); + htProcessJoint(htd, hand->kps[4], put_in_set, XRT_HAND_JOINT_THUMB_TIP); + + htProcessJoint(htd, index_metacarpal, put_in_set, XRT_HAND_JOINT_INDEX_METACARPAL); + htProcessJoint(htd, hand->kps[5], put_in_set, XRT_HAND_JOINT_INDEX_PROXIMAL); + htProcessJoint(htd, hand->kps[6], put_in_set, XRT_HAND_JOINT_INDEX_INTERMEDIATE); + htProcessJoint(htd, hand->kps[7], put_in_set, XRT_HAND_JOINT_INDEX_DISTAL); + htProcessJoint(htd, hand->kps[8], put_in_set, XRT_HAND_JOINT_INDEX_TIP); + + htProcessJoint(htd, middle_metacarpal, put_in_set, XRT_HAND_JOINT_MIDDLE_METACARPAL); + htProcessJoint(htd, hand->kps[9], put_in_set, XRT_HAND_JOINT_MIDDLE_PROXIMAL); + htProcessJoint(htd, hand->kps[10], put_in_set, XRT_HAND_JOINT_MIDDLE_INTERMEDIATE); + htProcessJoint(htd, hand->kps[11], put_in_set, XRT_HAND_JOINT_MIDDLE_DISTAL); + htProcessJoint(htd, hand->kps[12], put_in_set, XRT_HAND_JOINT_MIDDLE_TIP); + + htProcessJoint(htd, ring_metacarpal, put_in_set, XRT_HAND_JOINT_RING_METACARPAL); + htProcessJoint(htd, hand->kps[13], put_in_set, XRT_HAND_JOINT_RING_PROXIMAL); + htProcessJoint(htd, hand->kps[14], put_in_set, XRT_HAND_JOINT_RING_INTERMEDIATE); + htProcessJoint(htd, hand->kps[15], put_in_set, XRT_HAND_JOINT_RING_DISTAL); + htProcessJoint(htd, hand->kps[16], put_in_set, XRT_HAND_JOINT_RING_TIP); + + htProcessJoint(htd, pinky_metacarpal, put_in_set, XRT_HAND_JOINT_LITTLE_METACARPAL); + htProcessJoint(htd, hand->kps[17], put_in_set, XRT_HAND_JOINT_LITTLE_PROXIMAL); + htProcessJoint(htd, hand->kps[18], put_in_set, XRT_HAND_JOINT_LITTLE_INTERMEDIATE); + htProcessJoint(htd, hand->kps[19], put_in_set, XRT_HAND_JOINT_LITTLE_DISTAL); + htProcessJoint(htd, hand->kps[20], put_in_set, XRT_HAND_JOINT_LITTLE_TIP); + + put_in_set->is_active = true; + math_pose_identity(&put_in_set->hand_pose.pose); + + + put_in_set->hand_pose.pose.orientation = htd->stereo_camera_to_left_camera; + + put_in_set->hand_pose.relation_flags = valid_flags_ht; + + applyJointWidths(put_in_set); + applyJointOrientations(put_in_set, xr_indices[i]); + } + + htExitFrame(htd, false, final_hands_ordered_by_handedness, filtered_hands[0].timestamp); +} diff --git a/src/xrt/drivers/ht/ht_driver.c b/src/xrt/drivers/ht/ht_driver.c deleted file mode 100644 index 9df1a716f..000000000 --- a/src/xrt/drivers/ht/ht_driver.c +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2020, Collabora, Ltd. -// SPDX-License-Identifier: BSL-1.0 -/*! - * @file - * @brief Camera based hand tracking driver code. - * @author Christtoph Haag - * @ingroup drv_ht - */ - -#include "ht_driver.h" -#include "util/u_device.h" -#include "util/u_var.h" -#include "util/u_debug.h" -#include - -struct ht_device -{ - struct xrt_device base; - - struct xrt_tracked_hand *tracker; - - struct xrt_space_relation hand_relation[2]; - struct u_hand_tracking u_tracking[2]; - - struct xrt_tracking_origin tracking_origin; - - enum u_logging_level ll; -}; - -DEBUG_GET_ONCE_LOG_OPTION(ht_log, "HT_LOG", U_LOGGING_WARN) - -#define HT_TRACE(htd, ...) U_LOG_XDEV_IFL_T(&htd->base, htd->ll, __VA_ARGS__) -#define HT_DEBUG(htd, ...) U_LOG_XDEV_IFL_D(&htd->base, htd->ll, __VA_ARGS__) -#define HT_INFO(htd, ...) U_LOG_XDEV_IFL_I(&htd->base, htd->ll, __VA_ARGS__) -#define HT_WARN(htd, ...) U_LOG_XDEV_IFL_W(&htd->base, htd->ll, __VA_ARGS__) -#define HT_ERROR(htd, ...) U_LOG_XDEV_IFL_E(&htd->base, htd->ll, __VA_ARGS__) - -static inline struct ht_device * -ht_device(struct xrt_device *xdev) -{ - return (struct ht_device *)xdev; -} - -static void -ht_device_update_inputs(struct xrt_device *xdev) -{ - // Empty -} - -static void -ht_device_get_hand_tracking(struct xrt_device *xdev, - enum xrt_input_name name, - uint64_t at_timestamp_ns, - struct xrt_hand_joint_set *out_value) -{ - struct ht_device *htd = ht_device(xdev); - - enum xrt_hand hand; - int index; - - if (name == XRT_INPUT_GENERIC_HAND_TRACKING_LEFT) { - HT_TRACE(htd, "Get left hand tracking data"); - index = 0; - hand = XRT_HAND_LEFT; - } else if (name == XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT) { - HT_TRACE(htd, "Get right hand tracking data"); - index = 1; - hand = XRT_HAND_RIGHT; - } else { - HT_ERROR(htd, "unknown input name for hand tracker"); - return; - } - - - - htd->tracker->get_tracked_joints(htd->tracker, name, at_timestamp_ns, &htd->u_tracking[index].joints, - &htd->hand_relation[index]); - htd->u_tracking[index].timestamp_ns = at_timestamp_ns; - - struct xrt_pose identity = {.orientation = {.x = 0, .y = 0, .z = 0, .w = 1}, - .position = {.x = 0, .y = 0, .z = 0}}; - - u_hand_joints_set_out_data(&htd->u_tracking[index], hand, &htd->hand_relation[index], &identity, out_value); -} - -static void -ht_device_destroy(struct xrt_device *xdev) -{ - struct ht_device *htd = ht_device(xdev); - - // Remove the variable tracking. - u_var_remove_root(htd); - - u_device_free(&htd->base); -} - -struct xrt_device * -ht_device_create(struct xrt_auto_prober *xap, cJSON *attached_data, struct xrt_prober *xp) -{ - enum u_device_alloc_flags flags = U_DEVICE_ALLOC_NO_FLAGS; - - //! @todo 2 hands hardcoded - int num_hands = 2; - - struct ht_device *htd = U_DEVICE_ALLOCATE(struct ht_device, flags, num_hands, 0); - - - htd->base.tracking_origin = &htd->tracking_origin; - htd->base.tracking_origin->type = XRT_TRACKING_TYPE_RGB; - htd->base.tracking_origin->offset.position.x = 0.0f; - htd->base.tracking_origin->offset.position.y = 0.0f; - htd->base.tracking_origin->offset.position.z = 0.0f; - htd->base.tracking_origin->offset.orientation.w = 1.0f; - - htd->ll = debug_get_log_option_ht_log(); - - htd->base.update_inputs = ht_device_update_inputs; - htd->base.get_hand_tracking = ht_device_get_hand_tracking; - htd->base.destroy = ht_device_destroy; - - strncpy(htd->base.str, "Camera based Hand Tracker", XRT_DEVICE_NAME_LEN); - - htd->base.inputs[0].name = XRT_INPUT_GENERIC_HAND_TRACKING_LEFT; - htd->base.inputs[1].name = XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT; - - htd->base.name = XRT_DEVICE_HAND_TRACKER; - - if (xp->tracking->create_tracked_hand(xp->tracking, &htd->base, &htd->tracker) < 0) { - HT_ERROR(htd, "Failed to create hand tracker module"); - return NULL; - } - - u_hand_joints_init_default_set(&htd->u_tracking[XRT_HAND_LEFT], XRT_HAND_LEFT, XRT_HAND_TRACKING_MODEL_CAMERA, - 1.0); - u_hand_joints_init_default_set(&htd->u_tracking[XRT_HAND_RIGHT], XRT_HAND_RIGHT, XRT_HAND_TRACKING_MODEL_CAMERA, - 1.0); - - u_var_add_root(htd, "Camera based Hand Tracker", true); - u_var_add_ro_text(htd, htd->base.str, "Name"); - - HT_DEBUG(htd, "Hand Tracker initialized!"); - - return &htd->base; -} diff --git a/src/xrt/drivers/ht/ht_driver.cpp b/src/xrt/drivers/ht/ht_driver.cpp new file mode 100644 index 000000000..b398c6858 --- /dev/null +++ b/src/xrt/drivers/ht/ht_driver.cpp @@ -0,0 +1,778 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Camera based hand tracking driver code. + * @author Moses Turner + * @author Jakob Bornecrantz + * @ingroup drv_ht + */ + +#include "gstreamer/gst_pipeline.h" +#include "gstreamer/gst_sink.h" +#include "ht_interface.h" +#include "ht_driver.hpp" + +#include "../depthai/depthai_interface.h" + +#include "xrt/xrt_defines.h" +#include "xrt/xrt_frame.h" +#include "xrt/xrt_frameserver.h" + +#include "os/os_time.h" +#include "os/os_threading.h" + +#include "math/m_api.h" +#include "math/m_eigen_interop.hpp" + +#include "util/u_device.h" +#include "util/u_frame.h" +#include "util/u_sink.h" +#include "util/u_format.h" +#include "util/u_logging.h" +#include "util/u_time.h" +#include "util/u_trace_marker.h" +#include "util/u_time.h" +#include "util/u_json.h" +#include "util/u_config_json.h" + +#include "tracking/t_frame_cv_mat_wrapper.hpp" +#include "tracking/t_calibration_opencv.hpp" + +#include "templates/NaivePermutationSort.hpp" + +#include "ht_algorithm.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/*! + * Setup helper functions. + */ + +static bool +getCalibration(struct ht_device *htd, t_stereo_camera_calibration *calibration) +{ + xrt::auxiliary::tracking::StereoCameraCalibrationWrapper wrap(calibration); + xrt_vec3 trans = {(float)wrap.camera_translation_mat(0, 0), (float)wrap.camera_translation_mat(1, 0), + (float)wrap.camera_translation_mat(2, 0)}; + htd->baseline = m_vec3_len(trans); + +#if 0 + std::cout << "\n\nTRANSLATION VECTOR IS\n" << wrap.camera_translation_mat; + std::cout << "\n\nROTATION FROM LEFT TO RIGHT IS\n" << wrap.camera_rotation_mat << "\n"; +#endif + + cv::Matx34d P1; + cv::Matx34d P2; + + cv::Matx44d Q; + + // The only reason we're calling stereoRectify is because we want R1 and R2 for the + cv::stereoRectify(wrap.view[0].intrinsics_mat, // cameraMatrix1 + wrap.view[0].distortion_mat, // distCoeffs1 + wrap.view[1].intrinsics_mat, // cameraMatrix2 + wrap.view[1].distortion_mat, // distCoeffs2 + wrap.view[0].image_size_pixels_cv, // imageSize* + wrap.camera_rotation_mat, // R + wrap.camera_translation_mat, // T + htd->views[0].rotate_camera_to_stereo_camera, // R1 + htd->views[1].rotate_camera_to_stereo_camera, // R2 + P1, // P1 + P2, // P2 + Q, // Q + 0, // flags + -1.0f, // alpha + cv::Size(), // newImageSize + NULL, // validPixROI1 + NULL); // validPixROI2 + + //* Good enough guess that view 0 and view 1 are the same size. + + for (int i = 0; i < 2; i++) { + htd->views[i].cameraMatrix = wrap.view[i].intrinsics_mat; + + htd->views[i].distortion = wrap.view[i].distortion_fisheye_mat; + } + + htd->camera.one_view_size_px.w = wrap.view[0].image_size_pixels.w; + htd->camera.one_view_size_px.h = wrap.view[0].image_size_pixels.h; + + + cv::Matx33d rotate_stereo_camera_to_left_camera = htd->views[0].rotate_camera_to_stereo_camera.inv(); + + xrt_matrix_3x3 s; + s.v[0] = rotate_stereo_camera_to_left_camera(0, 0); + s.v[1] = rotate_stereo_camera_to_left_camera(0, 1); + s.v[2] = rotate_stereo_camera_to_left_camera(0, 2); + + s.v[3] = rotate_stereo_camera_to_left_camera(1, 0); + s.v[4] = rotate_stereo_camera_to_left_camera(1, 1); + s.v[5] = rotate_stereo_camera_to_left_camera(1, 2); + + s.v[6] = rotate_stereo_camera_to_left_camera(2, 0); + s.v[7] = rotate_stereo_camera_to_left_camera(2, 1); + s.v[8] = rotate_stereo_camera_to_left_camera(2, 2); + + xrt_quat tmp; + + math_quat_from_matrix_3x3(&s, &tmp); + + // Weird that I have to invert this quat, right? I think at some point - like probably just above this - I must + // have swapped row-major and col-major - remember, if you transpose a rotation matrix, you get its inverse. + // Doesn't matter that I don't understand - non-inverted looks definitely wrong, inverted looks definitely + // right. + math_quat_invert(&tmp, &htd->stereo_camera_to_left_camera); + +#if 0 + U_LOG_E("%f %f %f %f", htd->stereo_camera_to_left_camera.w, htd->stereo_camera_to_left_camera.x, + htd->stereo_camera_to_left_camera.y, htd->stereo_camera_to_left_camera.z); +#endif + + return true; +} + +static void +getStartupConfig(struct ht_device *htd, const cJSON *startup_config) +{ + const cJSON *palm_detection_type = u_json_get(startup_config, "palm_detection_model"); + const cJSON *keypoint_estimation_type = u_json_get(startup_config, "keypoint_estimation_model"); + const cJSON *uvc_wire_format = u_json_get(startup_config, "uvc_wire_format"); + + // IsString does its own null-checking + if (cJSON_IsString(palm_detection_type)) { + bool is_collabora = (strcmp(cJSON_GetStringValue(palm_detection_type), "collabora") == 0); + bool is_mediapipe = (strcmp(cJSON_GetStringValue(palm_detection_type), "mediapipe") == 0); + if (!is_collabora && !is_mediapipe) { + HT_WARN(htd, "Unknown palm detection type %s - should be \"collabora\" or \"mediapipe\"", + cJSON_GetStringValue(palm_detection_type)); + } + htd->startup_config.palm_detection_use_mediapipe = is_mediapipe; + } + + if (cJSON_IsString(keypoint_estimation_type)) { + bool is_collabora = (strcmp(cJSON_GetStringValue(keypoint_estimation_type), "collabora") == 0); + bool is_mediapipe = (strcmp(cJSON_GetStringValue(keypoint_estimation_type), "mediapipe") == 0); + if (!is_collabora && !is_mediapipe) { + HT_WARN(htd, "Unknown keypoint estimation type %s - should be \"collabora\" or \"mediapipe\"", + cJSON_GetStringValue(keypoint_estimation_type)); + } + htd->startup_config.keypoint_estimation_use_mediapipe = is_mediapipe; + } + + if (cJSON_IsString(uvc_wire_format)) { + bool is_yuv = (strcmp(cJSON_GetStringValue(uvc_wire_format), "yuv") == 0); + bool is_mjpeg = (strcmp(cJSON_GetStringValue(uvc_wire_format), "mjpeg") == 0); + if (!is_yuv && !is_mjpeg) { + HT_WARN(htd, "Unknown wire format type %s - should be \"yuv\" or \"mjpeg\"", + cJSON_GetStringValue(uvc_wire_format)); + } + if (is_yuv) { + HT_DEBUG(htd, "Using YUYV422!"); + htd->startup_config.desired_format = XRT_FORMAT_YUYV422; + } else { + HT_DEBUG(htd, "Using MJPEG!"); + htd->startup_config.desired_format = XRT_FORMAT_MJPEG; + } + } +} + +static void +getUserConfig(struct ht_device *htd) +{ + // The game here is to avoid bugs + be paranoid, not to be fast. If you see something that seems "slow" - don't + // fix it. Any of the tracking code is way stickier than this could ever be. + + struct u_config_json config_json = {}; + + u_config_json_open_or_create_main_file(&config_json); + if (!config_json.file_loaded) { + return; + } + + cJSON *ht_config_json = cJSON_GetObjectItemCaseSensitive(config_json.root, "config_ht"); + if (ht_config_json == NULL) { + return; + } + + // Don't get it twisted: initializing these to NULL is not cargo-culting. + // Uninitialized values on the stack aren't guaranteed to be 0, so these could end up pointing to what we + // *think* is a valid address but what is *not* one. + char *startup_config_string = NULL; + char *dynamic_config_string = NULL; + + { + const cJSON *startup_config_string_json = u_json_get(ht_config_json, "startup_config_index"); + if (cJSON_IsString(startup_config_string_json)) { + startup_config_string = cJSON_GetStringValue(startup_config_string_json); + } + + const cJSON *dynamic_config_string_json = u_json_get(ht_config_json, "dynamic_config_index"); + if (cJSON_IsString(dynamic_config_string_json)) { + dynamic_config_string = cJSON_GetStringValue(dynamic_config_string_json); + } + } + + if (startup_config_string != NULL) { + const cJSON *startup_config_obj = + u_json_get(u_json_get(ht_config_json, "startup_configs"), startup_config_string); + getStartupConfig(htd, startup_config_obj); + } + + if (dynamic_config_string != NULL) { + const cJSON *dynamic_config_obj = + u_json_get(u_json_get(ht_config_json, "dynamic_configs"), dynamic_config_string); + { + ht_dynamic_config *hdc = &htd->dynamic_config; + // Do the thing + u_json_get_string_into_array(u_json_get(dynamic_config_obj, "name"), hdc->name, 64); + + u_json_get_float(u_json_get(dynamic_config_obj, "hand_fc_min"), &hdc->hand_fc_min.val); + u_json_get_float(u_json_get(dynamic_config_obj, "hand_fc_min_d"), &hdc->hand_fc_min_d.val); + u_json_get_float(u_json_get(dynamic_config_obj, "hand_beta"), &hdc->hand_beta.val); + + u_json_get_float(u_json_get(dynamic_config_obj, "nms_iou"), &hdc->nms_iou.val); + u_json_get_float(u_json_get(dynamic_config_obj, "nms_threshold"), &hdc->nms_threshold.val); + + u_json_get_bool(u_json_get(dynamic_config_obj, "scribble_nms_detections"), + &hdc->scribble_nms_detections); + u_json_get_bool(u_json_get(dynamic_config_obj, "scribble_raw_detections"), + &hdc->scribble_raw_detections); + u_json_get_bool(u_json_get(dynamic_config_obj, "scribble_2d_keypoints"), + &hdc->scribble_2d_keypoints); + u_json_get_bool(u_json_get(dynamic_config_obj, "scribble_bounding_box"), + &hdc->scribble_bounding_box); + + U_LOG_E("Hey %s %s", dynamic_config_string, cJSON_Print(dynamic_config_obj)); + } + } + + + + cJSON_Delete(config_json.root); + return; +} + + +static void +userConfigSetDefaults(struct ht_device *htd) +{ + // Admit defeat: for now, Mediapipe's are still better than ours. + htd->startup_config.palm_detection_use_mediapipe = true; + htd->startup_config.keypoint_estimation_use_mediapipe = true; + + // Make sure you build DebugOptimized! + htd->startup_config.desired_format = XRT_FORMAT_YUYV422; + + + ht_dynamic_config *hdc = &htd->dynamic_config; + + hdc->scribble_nms_detections = true; + hdc->scribble_raw_detections = false; + hdc->scribble_2d_keypoints = true; + hdc->scribble_bounding_box = false; + + hdc->hand_fc_min.min = 0.0f; + hdc->hand_fc_min.max = 50.0f; + hdc->hand_fc_min.step = 0.05f; + hdc->hand_fc_min.val = FCMIN_HAND; + + hdc->hand_fc_min_d.min = 0.0f; + hdc->hand_fc_min_d.max = 50.0f; + hdc->hand_fc_min_d.step = 0.05f; + hdc->hand_fc_min_d.val = FCMIN_D_HAND; + + + hdc->hand_beta.min = 0.0f; + hdc->hand_beta.max = 50.0f; + hdc->hand_beta.step = 0.05f; + hdc->hand_beta.val = BETA_HAND; + + hdc->max_vel.min = 0.0f; + hdc->max_vel.max = 50.0f; + hdc->max_vel.step = 0.05f; + hdc->max_vel.val = 30.0f; // 30 m/s; about 108 kph. If your hand is going this fast, our tracking failing is the + // least of your problems. + + hdc->max_acc.min = 0.0f; + hdc->max_acc.max = 100.0f; + hdc->max_acc.step = 0.1f; + hdc->max_acc.val = 100.0f; // 100 m/s^2; about 10 Gs. Ditto. + + hdc->nms_iou.min = 0.0f; + hdc->nms_iou.max = 1.0f; + hdc->nms_iou.step = 0.01f; + + + hdc->nms_threshold.min = 0.0f; + hdc->nms_threshold.max = 1.0f; + hdc->nms_threshold.step = 0.01f; + + hdc->new_detection_threshold.min = 0.0f; + hdc->new_detection_threshold.max = 1.0f; + hdc->new_detection_threshold.step = 0.01f; + + + hdc->nms_iou.val = 0.05f; + hdc->nms_threshold.val = 0.3f; + hdc->new_detection_threshold.val = 0.6f; +} + + +static void +getModelsFolder(struct ht_device *htd) +{ +// Please bikeshed me on this! I don't know where is the best place to put this stuff! +#if 0 + char exec_location[1024] = {}; + readlink("/proc/self/exe", exec_location, 1024); + + HT_DEBUG(htd, "Exec at %s\n", exec_location); + + int end = 0; + while (exec_location[end] != '\0') { + HT_DEBUG(htd, "%d", end); + end++; + } + + while (exec_location[end] != '/' && end != 0) { + HT_DEBUG(htd, "%d %c", end, exec_location[end]); + exec_location[end] = '\0'; + end--; + } + + strcat(exec_location, "../share/monado/hand-tracking-models/"); + strcpy(htd->startup_config.model_slug, exec_location); +#else + const char *xdg_home = getenv("XDG_CONFIG_HOME"); + const char *home = getenv("HOME"); + if (xdg_home != NULL) { + strcpy(htd->startup_config.model_slug, xdg_home); + } else if (home != NULL) { + strcpy(htd->startup_config.model_slug, home); + } else { + assert(false); + } + strcat(htd->startup_config.model_slug, "/.local/share/monado/hand-tracking-models/"); +#endif +} + +#if defined(EXPERIMENTAL_DATASET_RECORDING) + +static void +htStartJsonCB(void *ptr) +{ + struct ht_device *htd = (struct ht_device *)ptr; + HT_INFO(htd, "Magic button pressed!"); + + // Wait for the hand tracker to be totally done with the current frame, then make it wait trying to relock this + // mutex for us to be done. + os_mutex_lock(&htd->unlocked_between_frames); + + if (htd->tracking_should_record_dataset == false) { + // Then we're starting up the pipeline. + HT_INFO(htd, "Starting dataset recording!"); + + + const char *source_name = "source_name"; + char pipeline_string[2048]; + + /* + None (0) – No preset + ultrafast (1) – ultrafast + superfast (2) – superfast + veryfast (3) – veryfast + faster (4) – faster + fast (5) – fast + medium (6) – medium + slow (7) – slow + slower (8) – slower + veryslow (9) – veryslow + placebo (10) – placebo + */ + +#if 0 + snprintf(pipeline_string, // + sizeof(pipeline_string), // + "appsrc name=\"%s\" ! " + "queue ! " + "videoconvert ! " + "queue ! " + "x264enc pass=qual quantizer=0 tune=film bitrate=\"%s\" speed-preset=\"%s\" ! " + "h264parse ! " + "queue ! " + "mp4mux ! " + "filesink location=\"%s\"", + source_name, "16384", "fast", "/tmp/moses.mp4"); +#elif 1 + snprintf(pipeline_string, // + sizeof(pipeline_string), // + "appsrc name=\"%s\" ! " + "queue ! " + "videoconvert ! " + "queue ! " + "x264enc pass=quant quantizer=20 tune=\"film\" speed-preset=\"veryfast\" ! " + "h264parse ! " + "queue ! " + "matroskamux ! " + "filesink location=\"%s\"", + source_name, "/tmp/moses.mkv"); +#elif 1 + snprintf(pipeline_string, // + sizeof(pipeline_string), // + "appsrc name=\"%s\" ! " + "queue ! " + "videoconvert ! " + "x265enc ! " + "h265parse ! " + "matroskamux ! " + "filesink location=\"%s\"", + source_name, "/tmp/moses.mkv"); +#endif + + gstreamer_pipeline_create_from_string(&htd->gst.xfctx, pipeline_string, &htd->gst.gp); + + gstreamer_sink_create_with_pipeline(htd->gst.gp, 2560, 800, XRT_FORMAT_R8G8B8, source_name, + &htd->gst.gs, &htd->gst.sink); + gstreamer_pipeline_play(htd->gst.gp); + + + htd->gst.output_root = cJSON_CreateObject(); + htd->gst.output_array = cJSON_CreateArray(); + cJSON_AddItemToObject(htd->gst.output_root, "hand_array", htd->gst.output_array); + + strcpy(htd->gui.start_json_record.label, "Stop recording and save dataset!"); + htd->gst.current_index = 0; + htd->tracking_should_record_dataset = true; + + } else { + // Then the pipeline was created sometime in the past and we have to destroy it + save everything to a + // file. + + gstreamer_pipeline_stop(htd->gst.gp); + + xrt_frame_context_destroy_nodes(&htd->gst.xfctx); + + + cJSON_AddNumberToObject(htd->gst.output_root, "num_frames", htd->gst.current_index); + cJSON_AddNumberToObject(htd->gst.output_root, "length_ns", htd->gst.last_frame_ns); + const char *string = cJSON_Print(htd->gst.output_root); + FILE *fp; + fp = fopen("/tmp/moses.json", "w"); + fprintf(fp, "%s", string); + fclose(fp); + cJSON_Delete(htd->gst.output_root); + + strcpy(htd->gui.start_json_record.label, "Start recording dataset!"); + htd->tracking_should_record_dataset = false; + } + + // We're done; let the hand tracker go about its business + os_mutex_unlock(&htd->unlocked_between_frames); +} +#endif + +static void +on_video_device(struct xrt_prober *xp, + struct xrt_prober_device *pdev, + const char *product, + const char *manufacturer, + const char *serial, + void *ptr) +{ + // Stolen from gui_scene_record + + struct ht_device *htd = (struct ht_device *)ptr; + + // Hardcoded for the Index. + if (product != NULL && manufacturer != NULL) { + if ((strcmp(product, "3D Camera") == 0) && (strcmp(manufacturer, "Etron Technology, Inc.") == 0)) { + xrt_prober_open_video_device(xp, pdev, &htd->camera.xfctx, &htd->camera.xfs); + return; + } + } +} + +/*! + * xrt_frame_sink function implementations + */ + +static void +ht_sink_push_frame(struct xrt_frame_sink *xs, struct xrt_frame *xf) +{ + XRT_TRACE_MARKER(); + struct ht_device *htd = container_of(xs, struct ht_device, sink); + assert(xf != NULL); + + if (!htd->tracking_should_die) { + os_mutex_lock(&htd->unlocked_between_frames); + + xrt_frame_reference(&htd->frame_for_process, xf); + htRunAlgorithm(htd); + xrt_frame_reference(&htd->frame_for_process, NULL); // Could let go of it a little earlier but nah + + os_mutex_unlock(&htd->unlocked_between_frames); + } +} + +/*! + * xrt_frame_node function implementations + */ + +static void +ht_node_break_apart(struct xrt_frame_node *node) +{ + struct ht_device *htd = container_of(node, struct ht_device, node); + HT_DEBUG(htd, "called!"); + // wrong but don't care +} + +static void +ht_node_destroy(struct xrt_frame_node *node) +{ + struct ht_device *htd = container_of(node, struct ht_device, node); + + HT_DEBUG(htd, "called!"); +} + +/*! + * xrt_device function implementations + */ + +static void +ht_device_update_inputs(struct xrt_device *xdev) +{ + // Empty +} + +static void +ht_device_get_hand_tracking(struct xrt_device *xdev, + enum xrt_input_name name, + uint64_t at_timestamp_ns, + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp_ns) +{ + struct ht_device *htd = ht_device(xdev); + + if (name != XRT_INPUT_GENERIC_HAND_TRACKING_LEFT && name != XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT) { + HT_ERROR(htd, "unknown input name for hand tracker"); + return; + } + bool hand_index = (name == XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT); // left=0, right=1 + + + + os_mutex_lock(&htd->openxr_hand_data_mediator); + memcpy(out_value, &htd->hands_for_openxr[hand_index], sizeof(struct xrt_hand_joint_set)); + // Instead of pose-predicting, we tell the caller that this joint set is a little old + *out_timestamp_ns = htd->hands_for_openxr_timestamp; + os_mutex_unlock(&htd->openxr_hand_data_mediator); +} + +static void +ht_device_destroy(struct xrt_device *xdev) +{ + struct ht_device *htd = ht_device(xdev); + HT_DEBUG(htd, "called!"); + + + xrt_frame_context_destroy_nodes(&htd->camera.xfctx); +#ifdef EXPERIMENTAL_DATASET_RECORDING + xrt_frame_context_destroy_nodes(&htd->gst.xfctx); +#endif + htd->tracking_should_die = true; + + // Lock this mutex so we don't try to free things as they're being used on the last iteration + os_mutex_lock(&htd->unlocked_between_frames); + destroyOnnx(htd); + // Remove the variable tracking. + u_var_remove_root(htd); + + // Shhhhhhhhhhh, it's okay. It'll all be okay. + htd->histories_3d.~vector(); + htd->views[0].bbox_histories.~vector(); + htd->views[1].bbox_histories.~vector(); + // Okay, fine, since we're mixing C and C++ idioms here, I couldn't find a clean way to implicitly + // call the destructors on these (ht_device doesn't have a destructor; neither do most of its members; and if + // you read u_device_allocate and u_device_free you'll agree it'd be somewhat annoying to write a + // constructor/destructor for ht_device), so we just manually call the destructors for things like std::vector's + // that need their destructors to be called to not leak. + + u_device_free(&htd->base); +} + +extern "C" struct xrt_device * +ht_device_create(struct xrt_prober *xp, struct t_stereo_camera_calibration *calib) +{ + enum ht_run_type run_type = HT_RUN_TYPE_VALVE_INDEX; + XRT_TRACE_MARKER(); + enum u_device_alloc_flags flags = U_DEVICE_ALLOC_NO_FLAGS; + + //! @todo 2 hands hardcoded + int num_hands = 2; + + // Allocate device + struct ht_device *htd = U_DEVICE_ALLOCATE(struct ht_device, flags, num_hands, 0); + + // Setup logging first. We like logging. + htd->ll = debug_get_log_option_ht_log(); + + /* + * Get configuration + */ + + assert(calib != NULL); + htd->run_type = run_type; + getCalibration(htd, calib); + // Set defaults - most people won't have a config json and it won't get past here. + userConfigSetDefaults(htd); + getUserConfig(htd); + getModelsFolder(htd); + + /* + * Add our xrt_frame_sink and xrt_frame_node implementations to ourselves + */ + + htd->sink.push_frame = &ht_sink_push_frame; + htd->node.break_apart = &ht_node_break_apart; + htd->node.destroy = &ht_node_destroy; + // Add ourselves to the frame context + xrt_frame_context_add(&htd->camera.xfctx, &htd->node); + + + + htd->camera.prober = xp; + htd->camera.xfs = NULL; // paranoia + + xrt_prober_list_video_devices(htd->camera.prober, on_video_device, htd); + + if (htd->camera.xfs == NULL) { + return NULL; + } + + + htd->views[0].htd = htd; + htd->views[1].htd = htd; // :) + + htd->views[0].view = 0; + htd->views[1].view = 1; + + + initOnnx(htd); + + htd->base.tracking_origin = &htd->tracking_origin; + htd->base.tracking_origin->type = XRT_TRACKING_TYPE_RGB; + htd->base.tracking_origin->offset.position.x = 0.0f; + htd->base.tracking_origin->offset.position.y = 0.0f; + htd->base.tracking_origin->offset.position.z = 0.0f; + htd->base.tracking_origin->offset.orientation.w = 1.0f; + + os_mutex_init(&htd->openxr_hand_data_mediator); + os_mutex_init(&htd->unlocked_between_frames); + + htd->base.update_inputs = ht_device_update_inputs; + htd->base.get_hand_tracking = ht_device_get_hand_tracking; + htd->base.destroy = ht_device_destroy; + + snprintf(htd->base.str, XRT_DEVICE_NAME_LEN, "Camera based Hand Tracker"); + snprintf(htd->base.serial, XRT_DEVICE_NAME_LEN, "Camera based Hand Tracker"); + + htd->base.inputs[0].name = XRT_INPUT_GENERIC_HAND_TRACKING_LEFT; + htd->base.inputs[1].name = XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT; + + // Yes, you need all of these. Yes, I tried disabling them all one at a time. You need all of these. + htd->base.name = XRT_DEVICE_HAND_TRACKER; + htd->base.device_type = XRT_DEVICE_TYPE_HAND_TRACKER; + htd->base.orientation_tracking_supported = true; + htd->base.position_tracking_supported = true; + htd->base.hand_tracking_supported = true; + + struct xrt_frame_sink *tmp = &htd->sink; + + + // This puts u_sink_create_to_r8g8b8_or_l8 on its own thread, so that nothing gets backed up if it runs slower + // than the native camera framerate. + u_sink_queue_create(&htd->camera.xfctx, tmp, &tmp); + + // Converts images (we'd expect YUV422 or MJPEG) to R8G8B8. Can take a long time, especially on unoptimized + // builds. If it's really slow, triple-check that you built Monado with optimizations! + u_sink_create_format_converter(&htd->camera.xfctx, XRT_FORMAT_R8G8B8, tmp, &tmp); + + // Puts the hand tracking code on its own thread, so that nothing upstream of it gets backed up if the hand + // tracking code runs slower than the upstream framerate. + u_sink_queue_create(&htd->camera.xfctx, tmp, &tmp); + + xrt_fs_mode *modes; + uint32_t count; + + xrt_fs_enumerate_modes(htd->camera.xfs, &modes, &count); + + // Index should only have XRT_FORMAT_YUYV422 or XRT_FORMAT_MJPEG. + + bool found_mode = false; + uint32_t selected_mode = 0; + + for (; selected_mode < count; selected_mode++) { + if (modes[selected_mode].format == htd->startup_config.desired_format) { + found_mode = true; + break; + } + } + + if (!found_mode) { + selected_mode = 0; + HT_WARN(htd, "Couldn't find desired camera mode! Something's probably wrong."); + } + + free(modes); + + u_var_add_root(htd, "Camera-based Hand Tracker", true); + + u_var_add_draggable_f32(htd, &htd->dynamic_config.hand_fc_min, "hand_fc_min"); + u_var_add_draggable_f32(htd, &htd->dynamic_config.hand_fc_min_d, "hand_fc_min_d"); + u_var_add_draggable_f32(htd, &htd->dynamic_config.hand_beta, "hand_beta"); + u_var_add_draggable_f32(htd, &htd->dynamic_config.nms_iou, "nms_iou"); + u_var_add_draggable_f32(htd, &htd->dynamic_config.nms_threshold, "nms_threshold"); + u_var_add_draggable_f32(htd, &htd->dynamic_config.new_detection_threshold, "new_detection_threshold"); + + u_var_add_bool(htd, &htd->dynamic_config.scribble_raw_detections, "Scribble raw detections"); + u_var_add_bool(htd, &htd->dynamic_config.scribble_nms_detections, "Scribble NMS detections"); + u_var_add_bool(htd, &htd->dynamic_config.scribble_2d_keypoints, "Scribble 2D keypoints"); + u_var_add_bool(htd, &htd->dynamic_config.scribble_bounding_box, "Scribble bounding box"); + +#ifdef EXPERIMENTAL_DATASET_RECORDING + htd->gui.start_json_record.ptr = htd; + htd->gui.start_json_record.cb = htStartJsonCB; + strcpy(htd->gui.start_json_record.label, "Start recording dataset!"); + u_var_add_button(htd, &htd->gui.start_json_record, ""); +#endif + + u_var_add_sink_debug(htd, &htd->debug_sink, "i"); + + xrt_fs_stream_start(htd->camera.xfs, tmp, XRT_FS_CAPTURE_TYPE_TRACKING, selected_mode); + + HT_DEBUG(htd, "Hand Tracker initialized!"); + + + return &htd->base; +} diff --git a/src/xrt/drivers/ht/ht_driver.h b/src/xrt/drivers/ht/ht_driver.h deleted file mode 100644 index 1cba10505..000000000 --- a/src/xrt/drivers/ht/ht_driver.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2020, Collabora, Ltd. -// SPDX-License-Identifier: BSL-1.0 -/*! - * @file - * @brief Interface to camera based hand tracking driver code. - * @author Christtoph Haag - * @ingroup drv_ht - */ - -#pragma once - -#include "math/m_api.h" -#include "xrt/xrt_device.h" -#include "xrt/xrt_prober.h" - -#ifdef __cplusplus -extern "C" { -#endif - -struct xrt_device * -ht_device_create(); - - -#ifdef __cplusplus -} -#endif diff --git a/src/xrt/drivers/ht/ht_driver.hpp b/src/xrt/drivers/ht/ht_driver.hpp new file mode 100644 index 000000000..9b10a8358 --- /dev/null +++ b/src/xrt/drivers/ht/ht_driver.hpp @@ -0,0 +1,380 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Defines and common includes for camera-based hand tracker + * @author Moses Turner + * @ingroup drv_ht + */ + +#pragma once + +#include "ht_interface.h" +#include "os/os_threading.h" + +#include "xrt/xrt_device.h" +#include "xrt/xrt_prober.h" +#include "xrt/xrt_frame.h" +#include "xrt/xrt_frameserver.h" + +#include "math/m_api.h" +#include "math/m_vec3.h" +#include "math/m_filter_one_euro.h" + +#include "util/u_var.h" +#include "util/u_json.h" +#include "util/u_sink.h" +#include "util/u_debug.h" +#include "util/u_device.h" + +#include "util/u_template_historybuf.hpp" + +#ifdef XRT_HAVE_GST +#include "gstreamer/gst_pipeline.h" +#include "gstreamer/gst_sink.h" +#endif + +#include + +#include "core/session/onnxruntime_c_api.h" + +#include +#include + +using namespace xrt::auxiliary::util; + +DEBUG_GET_ONCE_LOG_OPTION(ht_log, "HT_LOG", U_LOGGING_WARN) + +#define HT_TRACE(htd, ...) U_LOG_XDEV_IFL_T(&htd->base, htd->ll, __VA_ARGS__) +#define HT_DEBUG(htd, ...) U_LOG_XDEV_IFL_D(&htd->base, htd->ll, __VA_ARGS__) +#define HT_INFO(htd, ...) U_LOG_XDEV_IFL_I(&htd->base, htd->ll, __VA_ARGS__) +#define HT_WARN(htd, ...) U_LOG_XDEV_IFL_W(&htd->base, htd->ll, __VA_ARGS__) +#define HT_ERROR(htd, ...) U_LOG_XDEV_IFL_E(&htd->base, htd->ll, __VA_ARGS__) + +// #define ht_ + + +// To make clang-tidy happy +#define opencv_distortion_param_num 4 + +/* + * + * Compile-time defines to choose where to get camera frames from and what kind of output to give out + * + */ +#undef EXPERIMENTAL_DATASET_RECORDING + +#define FCMIN_BBOX_ORIENTATION 3.0f +#define FCMIN_D_BB0X_ORIENTATION 10.0f +#define BETA_BB0X_ORIENTATION 0.0f + +// #define FCMIN_BBOX_POSITION 15.0f +// #define FCMIN_D_BB0X_POSITION 12.0f +// #define BETA_BB0X_POSITION 0.3f + +#define FCMIN_BBOX_POSITION 30.0f +#define FCMIN_D_BB0X_POSITION 25.0f +#define BETA_BB0X_POSITION 0.6f + + + +#define FCMIN_HAND 4.0f +#define FCMIN_D_HAND 12.0f +#define BETA_HAND 0.05f + +#ifdef __cplusplus +extern "C" { +#endif + +enum HandJoint7Keypoint +{ + WRIST_7KP = 0, + INDEX_7KP = 1, + MIDDLE_7KP = 2, + RING_7KP = 3, + LITTLE_7KP = 4, + THUMB_METACARPAL_7KP = 5, + THMB_PROXIMAL_7KP = 6 +}; + +enum HandJoint21Keypoint +{ + WRIST = 0, + + THMB_MCP = 1, + THMB_PXM = 2, + THMB_DST = 3, + THMB_TIP = 4, + + INDX_PXM = 5, + INDX_INT = 6, + INDX_DST = 7, + INDX_TIP = 8, + + MIDL_PXM = 9, + MIDL_INT = 10, + MIDL_DST = 11, + MIDL_TIP = 12, + + RING_PXM = 13, + RING_INT = 14, + RING_DST = 15, + RING_TIP = 16, + + LITL_PXM = 17, + LITL_INT = 18, + LITL_DST = 19, + LITL_TIP = 20 +}; + +struct Palm7KP +{ + struct xrt_vec2 kps[7]; + float confidence; // BETWEEN 0 and 1. okay???? okay????!??? +}; + +struct DetectionModelOutput +{ + float rotation; + float size; + xrt_vec2 center; + Palm7KP palm; + + cv::Matx23f warp_there; + cv::Matx23f warp_back; +}; + +// To keep you on your toes. *Don't* think the 2D hand is the same as the 3D! +struct Hand2D +{ + struct xrt_vec3 kps[21]; + // Third value is depth from ML model. Do not believe the depth. +}; + +struct Hand3D +{ + struct xrt_vec3 kps[21]; + float y_disparity_error; + float flow_error; + int idx_l; + int idx_r; + bool rejected_by_smush; // init to false. + + float handedness; + uint64_t timestamp; +}; + + +struct HandHistory3D +{ + // Index 0 is current frame, index 1 is last frame, index 2 is second to last frame. + // No particular reason to keep the last 5 frames. we only really only use the current and last one. + float handedness; + bool have_prev_hand = false; + double prev_dy; + uint64_t prev_ts_for_alpha; // also in last_hands_unfiltered[0] but go away. + + uint64_t first_ts; + uint64_t prev_filtered_ts; + + HistoryBuffer last_hands_unfiltered; + HistoryBuffer last_hands_filtered; + + // Euro filter for 21kps. + m_filter_euro_vec3 filters[21]; + int uuid; +}; + +struct HandHistory2DBBox +{ + // Ugh, I should definitely iterate these somehow... + // m_filter_euro_vec2 m_filter_wrist; + // m_filter_euro_vec2 m_filter_index; + // m_filter_euro_vec2 m_filter_middle; + // m_filter_euro_vec2 m_filter_pinky; + + m_filter_euro_vec2 m_filter_center; + m_filter_euro_vec2 m_filter_direction; + + HistoryBuffer wrist_unfiltered; + HistoryBuffer index_unfiltered; + HistoryBuffer middle_unfiltered; + HistoryBuffer pinky_unfiltered; + bool htAlgorithm_approves = false; +}; + + +struct ModelInfo +{ + OrtSession *session = nullptr; + OrtMemoryInfo *memoryInfo = nullptr; + // std::vector's don't make too much sense here, but they're oh so easy + std::vector input_shape; + size_t input_size_bytes; + std::vector output_names; + std::vector input_names; +}; + + +// Forward declaration for ht_view +struct ht_device; + +struct ht_view +{ + ht_device *htd; + int view; // :))) + + // Loaded from config file + cv::Matx distortion; + cv::Matx cameraMatrix; + cv::Matx33d rotate_camera_to_stereo_camera; // R1 or R2 + + cv::Mat run_model_on_this; + cv::Mat debug_out_to_this; + + std::vector bbox_histories; + + struct ModelInfo detection_model; + std::vector (*run_detection_model)(struct ht_view *htv, cv::Mat &img); + + struct ModelInfo keypoint_model; + // The cv::mat is passed by value, *not* passed by reference or by pointer; + // in the tight loop that sets these off we reuse that cv::Mat; changing the data pointer as all the models are + // running is... going to wreak havoc let's say that. + Hand2D (*run_keypoint_model)(struct ht_view *htv, cv::Mat img); +}; + +enum ht_detection_scribble +{ + HT_DETECTION_SCRIBBLE_ALL, + HT_DETECTION_SCRIBBLE_SOME, + HT_DETECTION_SCRIBBLE_NONE +}; + +struct ht_dynamic_config +{ + char name[64]; + struct u_var_draggable_f32 hand_fc_min; + struct u_var_draggable_f32 hand_fc_min_d; + struct u_var_draggable_f32 hand_beta; + struct u_var_draggable_f32 max_vel; + struct u_var_draggable_f32 max_acc; + struct u_var_draggable_f32 nms_iou; + struct u_var_draggable_f32 nms_threshold; + struct u_var_draggable_f32 new_detection_threshold; + bool scribble_raw_detections; + bool scribble_nms_detections; + bool scribble_2d_keypoints; + bool scribble_bounding_box; +}; + +struct ht_startup_config +{ + bool palm_detection_use_mediapipe = false; + bool keypoint_estimation_use_mediapipe = false; + enum xrt_format desired_format; + char model_slug[1024]; +}; + +// This is all ad-hoc! Review very welcome! +struct ht_device +{ + struct xrt_device base; + + struct xrt_tracking_origin tracking_origin; // probably cargo-culted + + struct xrt_frame_sink sink; + struct xrt_frame_node node; + + struct u_sink_debug debug_sink; // this must be bad. + + + struct + { + struct xrt_frame_context xfctx; + + struct xrt_fs *xfs; + + struct xrt_fs_mode mode; + + struct xrt_prober *prober; + + struct xrt_size one_view_size_px; + } camera; + + + +#if defined(EXPERIMENTAL_DATASET_RECORDING) + struct + { + struct u_var_button start_json_record; + } gui; + struct + { + struct gstreamer_pipeline *gp; + struct gstreamer_sink *gs; + struct xrt_frame_sink *sink; + struct xrt_frame_context xfctx; + uint64_t offset_ns; + uint64_t last_frame_ns; + uint64_t current_index; + + cJSON *output_root; + cJSON *output_array; + } gst; +#endif + + + + const OrtApi *ort_api; + OrtEnv *ort_env; + + struct xrt_frame *frame_for_process; + cv::Mat *mat_for_process; + + struct ht_view views[2]; + + float baseline; + struct xrt_quat stereo_camera_to_left_camera; + + uint64_t current_frame_timestamp; // SUPER dumb. + + std::vector histories_3d; + + struct os_mutex openxr_hand_data_mediator; + struct xrt_hand_joint_set hands_for_openxr[2]; + uint64_t hands_for_openxr_timestamp; + + // Only change these when you have unlocked_between_frames, ie. when the hand tracker is between frames. + bool tracking_should_die; + bool tracking_should_record_dataset; + struct os_mutex unlocked_between_frames; + + // Change this whenever you want + bool debug_scribble = true; + + ht_run_type run_type; + + + + struct ht_startup_config startup_config; + struct ht_dynamic_config dynamic_config; + + + int dynamic_config_to_use; + + + + enum u_logging_level ll; +}; + +static inline struct ht_device * +ht_device(struct xrt_device *xdev) +{ + return (struct ht_device *)xdev; +} + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/ht/ht_hand_math.hpp b/src/xrt/drivers/ht/ht_hand_math.hpp new file mode 100644 index 000000000..00b3c2870 --- /dev/null +++ b/src/xrt/drivers/ht/ht_hand_math.hpp @@ -0,0 +1,431 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Helper math to do things with 3D hands for the camera-based hand tracker + * @author Moses Turner + * @author Nick Klingensmith + * @ingroup drv_ht + */ + +#pragma once + +#include "math/m_api.h" +#include "math/m_vec3.h" + +#include "ht_driver.hpp" +#include "util/u_time.h" +#include "xrt/xrt_defines.h" + + +const int num_real_joints = 21; + +static float +errHandDisparity(Hand2D *left_rays, Hand2D *right_rays) +{ + float error_y_diff = 0.0f; + for (int i = 0; i < 21; i++) { + float diff_y = fabsf(left_rays->kps[i].y - right_rays->kps[i].y); + // Big question about what's the best loss function. Gut feeling was "I should be using sum of squared + // errors" but I don't really know. Using just sum of errors for now. Ideally it'd also be not very + // sensitive to one or two really bad outliers. + error_y_diff += diff_y; + } + // U_LOG_E("stereo camera err is %f, y_disparity is %f", err_stereo_camera, error_y_diff); + return error_y_diff; +} + +static float +sumOfHandJointDistances(Hand3D *one, Hand3D *two) +{ + float dist = 0.0f; + for (int i = 0; i < num_real_joints; i++) { + dist += m_vec3_len(one->kps[i] - two->kps[i]); + } + return dist; +} + +static float +errHandHistory(HandHistory3D *history_hand, Hand3D *present_hand) +{ + // Remember we never have to deal with an empty hand. Can always access the last element. + return sumOfHandJointDistances(history_hand->last_hands_unfiltered[0], present_hand); +} + + +static void +applyJointWidths(struct xrt_hand_joint_set *set) +{ + // Thanks to Nick Klingensmith for this idea + struct xrt_hand_joint_value *gr = set->values.hand_joint_set_default; + + const float finger_joint_size[5] = {0.022f, 0.021f, 0.022f, 0.021f, 0.02f}; + const float hand_finger_size[5] = {1.0f, 1.0f, 0.83f, 0.75f}; + + const float thumb_size[4] = {0.016f, 0.014f, 0.012f, 0.012f}; + float mul = 1.0f; + + for (int i = XRT_HAND_JOINT_THUMB_METACARPAL; i <= XRT_HAND_JOINT_THUMB_TIP; i++) { + int j = i - XRT_HAND_JOINT_THUMB_METACARPAL; + gr[i].radius = thumb_size[j] * mul; + } + + for (int finger = 0; finger < 4; finger++) { + for (int joint = 0; joint < 5; joint++) { + int set_idx = finger * 5 + joint + XRT_HAND_JOINT_INDEX_METACARPAL; + float val = finger_joint_size[joint] * hand_finger_size[finger] * .5 * mul; + gr[set_idx].radius = val; + } + } + // The radius of each joint is the distance from the joint to the skin in meters. -OpenXR spec. + set->values.hand_joint_set_default[XRT_HAND_JOINT_PALM].radius = + .032f * .5f; // Measured my palm thickness with calipers + set->values.hand_joint_set_default[XRT_HAND_JOINT_WRIST].radius = + .040f * .5f; // Measured my wrist thickness with calipers +} + +static void +applyThumbIndexDrag(Hand3D *hand) +{ + // TERRIBLE HACK. + // Puts the thumb and pointer a bit closer together to be better at triggering XR clients' pinch detection. + const float max_radius = 0.05; + const float min_radius = 0.00; + + // no min drag, min drag always 0. + const float max_drag = 0.85f; + + xrt_vec3 thumb = hand->kps[THMB_TIP]; + xrt_vec3 index = hand->kps[INDX_TIP]; + xrt_vec3 ttp = index - thumb; + float length = m_vec3_len(ttp); + if ((length > max_radius)) { + return; + } + + + float amount = math_map_ranges(length, min_radius, max_radius, max_drag, 0.0f); + + hand->kps[THMB_TIP] = m_vec3_lerp(thumb, index, amount * 0.5f); + hand->kps[INDX_TIP] = m_vec3_lerp(index, thumb, amount * 0.5f); +} + +static void +applyJointOrientations(struct xrt_hand_joint_set *set, bool is_right) +{ + // The real rule to follow is that each joint's "X" axis is along the axis along which it can bend. + // The nature of our estimation makes this a bit difficult, but these should work okay-ish under perfect + // conditions + if (set->is_active == false) { + return; + } + +#define gl(jt) set->values.hand_joint_set_default[jt].relation.pose.position + + xrt_vec3 pinky_prox = gl(XRT_HAND_JOINT_LITTLE_PROXIMAL); + + xrt_vec3 index_prox = gl(XRT_HAND_JOINT_INDEX_PROXIMAL); + + + xrt_vec3 pinky_to_index_prox = m_vec3_normalize(index_prox - pinky_prox); + if (is_right) { + pinky_to_index_prox = m_vec3_mul_scalar(pinky_to_index_prox, -1.0f); + } + + std::vector> fingers_with_joints_in_them = { + + {XRT_HAND_JOINT_INDEX_METACARPAL, XRT_HAND_JOINT_INDEX_PROXIMAL, XRT_HAND_JOINT_INDEX_INTERMEDIATE, + XRT_HAND_JOINT_INDEX_DISTAL, XRT_HAND_JOINT_INDEX_TIP}, + + {XRT_HAND_JOINT_MIDDLE_METACARPAL, XRT_HAND_JOINT_MIDDLE_PROXIMAL, XRT_HAND_JOINT_MIDDLE_INTERMEDIATE, + XRT_HAND_JOINT_MIDDLE_DISTAL, XRT_HAND_JOINT_MIDDLE_TIP}, + + {XRT_HAND_JOINT_RING_METACARPAL, XRT_HAND_JOINT_RING_PROXIMAL, XRT_HAND_JOINT_RING_INTERMEDIATE, + XRT_HAND_JOINT_RING_DISTAL, XRT_HAND_JOINT_RING_TIP}, + + {XRT_HAND_JOINT_LITTLE_METACARPAL, XRT_HAND_JOINT_LITTLE_PROXIMAL, XRT_HAND_JOINT_LITTLE_INTERMEDIATE, + XRT_HAND_JOINT_LITTLE_DISTAL, XRT_HAND_JOINT_LITTLE_TIP}, + + }; + for (std::vector finger : fingers_with_joints_in_them) { + + for (int i = 0; i < 4; i++) { + // Don't do fingertips. (Fingertip would be index 4.) + struct xrt_vec3 forwards = m_vec3_normalize(gl(finger[i + 1]) - gl(finger[i])); + struct xrt_vec3 backwards = m_vec3_mul_scalar(forwards, -1.0f); + + struct xrt_vec3 left = m_vec3_orthonormalize(forwards, pinky_to_index_prox); + // float dot = m_vec3_dot(backwards, left); + // assert((m_vec3_dot(backwards,left) == 0.0f)); + math_quat_from_plus_x_z( + &left, &backwards, + &set->values.hand_joint_set_default[finger[i]].relation.pose.orientation); + } + // Do fingertip! Per XR_EXT_hand_tracking, just copy the distal joint's orientation. Doing anything else + // is wrong. + set->values.hand_joint_set_default[finger[4]].relation.pose.orientation = + set->values.hand_joint_set_default[finger[3]].relation.pose.orientation; + } + + // wrist! + // Not the best but acceptable. Eventually, probably, do triangle of wrist pinky prox and index prox. + set->values.hand_joint_set_default[XRT_HAND_JOINT_WRIST].relation.pose.orientation = + set->values.hand_joint_set_default[XRT_HAND_JOINT_MIDDLE_METACARPAL].relation.pose.orientation; + + + // palm! + set->values.hand_joint_set_default[XRT_HAND_JOINT_PALM].relation.pose.orientation = + set->values.hand_joint_set_default[XRT_HAND_JOINT_MIDDLE_METACARPAL].relation.pose.orientation; + + // thumb! + // When I look at Ultraleap tracking, there's like, a "plane" made by the tip, distal and proximal (and kinda + // MCP, but least squares fitting a plane is too hard for my baby brain) Normal to this plane is the +X, and + // obviously forwards to the next joint is the -Z. + xrt_vec3 thumb_prox_to_dist = gl(XRT_HAND_JOINT_THUMB_DISTAL) - gl(XRT_HAND_JOINT_THUMB_PROXIMAL); + xrt_vec3 thumb_dist_to_tip = gl(XRT_HAND_JOINT_THUMB_TIP) - gl(XRT_HAND_JOINT_THUMB_DISTAL); + xrt_vec3 plane_normal; + if (!is_right) { + math_vec3_cross(&thumb_prox_to_dist, &thumb_dist_to_tip, &plane_normal); + } else { + math_vec3_cross(&thumb_dist_to_tip, &thumb_prox_to_dist, &plane_normal); + } + std::vector thumbs = {XRT_HAND_JOINT_THUMB_METACARPAL, XRT_HAND_JOINT_THUMB_PROXIMAL, + XRT_HAND_JOINT_THUMB_DISTAL, XRT_HAND_JOINT_THUMB_TIP}; + for (int i = 0; i < 3; i++) { + struct xrt_vec3 backwards = + m_vec3_mul_scalar(m_vec3_normalize(gl(thumbs[i + 1]) - gl(thumbs[i])), -1.0f); + + struct xrt_vec3 left = m_vec3_orthonormalize(backwards, plane_normal); + math_quat_from_plus_x_z(&left, &backwards, + &set->values.hand_joint_set_default[thumbs[i]].relation.pose.orientation); + } + struct xrt_quat *tip = &set->values.hand_joint_set_default[XRT_HAND_JOINT_THUMB_TIP].relation.pose.orientation; + struct xrt_quat *distal = + &set->values.hand_joint_set_default[XRT_HAND_JOINT_THUMB_DISTAL].relation.pose.orientation; + memcpy(tip, distal, sizeof(struct xrt_quat)); +} + +static float +handednessJointSet(Hand3D *set) +{ + // Guess if hand is left or right. + // Left is negative, right is positive. + + + // xrt_vec3 middle_mcp = gl(XRT_HAND_JOINT_MIDDLE_METACARPAL); + + xrt_vec3 pinky_prox = set->kps[LITL_PXM]; // gl(XRT_HAND_JOINT_LITTLE_PROXIMAL); + + xrt_vec3 index_prox = set->kps[INDX_PXM]; // gl(XRT_HAND_JOINT_INDEX_PROXIMAL); + + xrt_vec3 pinky_to_index_prox = m_vec3_normalize(index_prox - pinky_prox); + + float handedness = 0.0f; + + for (int i : {INDX_PXM, MIDL_PXM, RING_PXM, LITL_PXM}) { + xrt_vec3 prox = set->kps[i]; + xrt_vec3 intr = set->kps[i + 1]; + xrt_vec3 dist = set->kps[i + 2]; + xrt_vec3 tip = set->kps[i + 3]; + + xrt_vec3 prox_to_int = m_vec3_normalize(intr - prox); + xrt_vec3 int_to_dist = m_vec3_normalize(dist - intr); + xrt_vec3 dist_to_tip = m_vec3_normalize(tip - dist); + + xrt_vec3 checks[2]; + + math_vec3_cross(&prox_to_int, &int_to_dist, &checks[0]); + math_vec3_cross(&int_to_dist, &dist_to_tip, &checks[1]); + + handedness += m_vec3_dot(m_vec3_normalize(pinky_to_index_prox), (checks[0])); + handedness += m_vec3_dot(m_vec3_normalize(pinky_to_index_prox), (checks[1])); + } + set->handedness = handedness / (4 * 2); + return set->handedness; +} + +static void +handednessHandHistory3D(HandHistory3D *history) +{ + + float inter = handednessJointSet(history->last_hands_unfiltered[0]); + + if ((fabsf(inter) > 0.3f) || (fabsf(history->handedness) < 0.3f)) { + history->handedness += inter; + } + const int max_handedness = 2.0f; + if (history->handedness > max_handedness) { + history->handedness = max_handedness; + } else if (history->handedness < -max_handedness) { + history->handedness = -max_handedness; + } +} + +static void +handEuroFiltersInit(HandHistory3D *history, double fc_min, double fc_min_d, double beta) +{ + for (int i = 0; i < 21; i++) { + m_filter_euro_vec3_init(&history->filters[i], fc_min, beta, fc_min_d); + } +} + +static double +calc_smoothing_alpha(double Fc, double dt) +{ + /* Calculate alpha = (1 / (1 + tau/dt)) where tau = 1.0 / (2 * pi * Fc), + * this is a straight rearrangement with fewer divisions */ + double r = 2.0 * M_PI * Fc * dt; + return r / (r + 1.0); +} + +static double +exp_smooth(double alpha, double y, double prev_y) +{ + return alpha * y + (1.0 - alpha) * prev_y; +} + +void +handEuroFiltersRun(struct ht_device *htd, HandHistory3D *f, Hand3D *out_hand) +{ + // Assume present hand is in element 0! +#if 0 + // float vals[4] = {0.5, 0.33, 0.1, 0.07}; + float vals[4] = {0.9, 0.09, 0.009, 0.001}; + int m = f->last_hands_unfiltered.length-1; + double ts_out = (vals[0] * (double)f->last_hands_unfiltered[std::min(m,0)]->timestamp) + + (vals[1] * (double)f->last_hands_unfiltered[std::min(m,1)]->timestamp) + + (vals[2] * (double)f->last_hands_unfiltered[std::min(m,2)]->timestamp) + + (vals[3] * (double)f->last_hands_unfiltered[std::min(m,3)]->timestamp); + out_hand->timestamp = (uint64_t)ts_out; + + for (int kp_idx = 0; kp_idx < 21; kp_idx++) { + for (int hist_idx = 0; hist_idx < 4; hist_idx++) { + float *in_y_arr = (float *)&f->last_hands_unfiltered[std::min(m,hist_idx)]->kps[kp_idx]; + float *out_y_arr = (float *)&out_hand->kps[kp_idx]; + for (int i = 0; i < 3; i++) { + out_y_arr[i] += in_y_arr[i] * vals[hist_idx]; + } + } + } +#elif 0 + for (int i = 0; i < 21; i++) { + m_filter_euro_vec3_run(&f->filters[i], f->last_hands_unfiltered[0]->timestamp, + &f->last_hands_unfiltered[0]->kps[i], &out_hand->kps[i]); + } + // conspicuously wrong! + out_hand->timestamp = f->last_hands_unfiltered[0]->timestamp; +#else + + if (!f->have_prev_hand) { + f->last_hands_filtered.push(*f->last_hands_unfiltered[0]); + uint64_t ts = f->last_hands_unfiltered[0]->timestamp; + f->prev_ts_for_alpha = ts; + f->first_ts = ts; + f->prev_filtered_ts = ts; + f->prev_dy = 0; + f->have_prev_hand = true; + *out_hand = *f->last_hands_unfiltered[0]; + } + uint64_t ts = f->last_hands_unfiltered[0]->timestamp; + double dt, alpha_d; + dt = (double)(ts - f->prev_ts_for_alpha) / U_TIME_1S_IN_NS; + + double abs_dy = + (sumOfHandJointDistances(f->last_hands_unfiltered[0], f->last_hands_filtered[0]) / 21.0f) * 0.7f; + alpha_d = calc_smoothing_alpha(htd->dynamic_config.hand_fc_min_d.val, dt); + + double alpha, fc_cutoff; + f->prev_dy = exp_smooth(alpha_d, abs_dy, f->prev_dy); + + fc_cutoff = htd->dynamic_config.hand_fc_min.val + htd->dynamic_config.hand_beta.val * f->prev_dy; + alpha = calc_smoothing_alpha(fc_cutoff, dt); + HT_DEBUG(htd, "dt is %f, abs_dy is %f, alpha is %f", dt, abs_dy, alpha); + + for (int i = 0; i < 21; i++) { + out_hand->kps[i].x = + exp_smooth(alpha, f->last_hands_unfiltered[0]->kps[i].x, f->last_hands_filtered[0]->kps[i].x); + out_hand->kps[i].y = + exp_smooth(alpha, f->last_hands_unfiltered[0]->kps[i].y, f->last_hands_filtered[0]->kps[i].y); + out_hand->kps[i].z = + exp_smooth(alpha, f->last_hands_unfiltered[0]->kps[i].z, f->last_hands_filtered[0]->kps[i].z); + } + double prev_ts_offset = (double)(f->prev_filtered_ts - f->first_ts); + double current_ts_offset = (double)(ts - f->first_ts); + double new_filtered_ts_offset = exp_smooth(alpha, current_ts_offset, prev_ts_offset); + uint64_t new_filtered_ts = (uint64_t)(new_filtered_ts_offset) + f->first_ts; + out_hand->timestamp = new_filtered_ts; + f->prev_filtered_ts = out_hand->timestamp; + f->prev_ts_for_alpha = ts; // NOT the filtered timestamp. NO. +#endif +} + +static bool +rejectTooFar(struct ht_device *htd, Hand3D *hand) +{ + const float max_dist = 1.0f; // this sucks too - make it bigger if you can. + const float max_dist_from_camera_sqrd = max_dist * max_dist; + for (int i = 0; i < 21; i++) { + xrt_vec3 pos = hand->kps[i]; + float len = m_vec3_len_sqrd(pos); // Faster. + if (len > max_dist_from_camera_sqrd) { + goto reject; + } + } + return true; + +reject: + HT_TRACE(htd, "Rejected too far!"); + return false; +} + +static bool +rejectTooClose(struct ht_device *htd, Hand3D *hand) +{ + const float min_dist = 0.12f; // Be a bit aggressive here - it's nice to not let people see our tracking fail + // when the hands are way too close + const float min_dist_from_camera_sqrd = min_dist * min_dist; + + for (int i = 0; i < 21; i++) { + xrt_vec3 pos = hand->kps[i]; + float len = m_vec3_len_sqrd(pos); // Faster. + if (len < min_dist_from_camera_sqrd) { + goto reject; + } + if (pos.z > min_dist) { // remember negative-Z is forward! + goto reject; + } + } + return true; + +reject: + HT_TRACE(htd, "Rejected too close!"); + return false; +} + +static bool +rejectTinyPalm(struct ht_device *htd, Hand3D *hand) +{ + // This one sucks, because some people really have tiny hands. If at some point you can stop using it, stop + // using it. + // Weird scoping so that we can still do gotos + + { + float len = m_vec3_len(hand->kps[WRIST] - hand->kps[INDX_PXM]); + if ((len < 0.03f || len > 0.25f)) { + goto reject; + } + } + + { + float len = m_vec3_len(hand->kps[WRIST] - hand->kps[MIDL_PXM]); + if (len < 0.03f || len > 0.25f) { + goto reject; + } + } + + return true; + +reject: + HT_TRACE(htd, "Rejected because too big or too small!"); + return false; +} diff --git a/src/xrt/drivers/ht/ht_image_math.hpp b/src/xrt/drivers/ht/ht_image_math.hpp new file mode 100644 index 000000000..0c1444aae --- /dev/null +++ b/src/xrt/drivers/ht/ht_image_math.hpp @@ -0,0 +1,268 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Helper math to do things with images for the camera-based hand tracker + * @author Moses Turner + * @ingroup drv_ht + */ + +#pragma once + +#include "xrt/xrt_defines.h" +#include "math/m_api.h" +#include "math/m_vec2.h" +#include "math/m_vec3.h" + +#include "ht_driver.hpp" + +#include +#include +#include +#include + + +static cv::Scalar +hsv2rgb(float fH, float fS, float fV) +{ + float fC = fV * fS; // Chroma + float fHPrime = fmod(fH / 60.0, 6); + float fX = fC * (1 - fabs(fmod(fHPrime, 2) - 1)); + float fM = fV - fC; + + float fR, fG, fB; + + if (0 <= fHPrime && fHPrime < 1) { + fR = fC; + fG = fX; + fB = 0; + } else if (1 <= fHPrime && fHPrime < 2) { + fR = fX; + fG = fC; + fB = 0; + } else if (2 <= fHPrime && fHPrime < 3) { + fR = 0; + fG = fC; + fB = fX; + } else if (3 <= fHPrime && fHPrime < 4) { + fR = 0; + fG = fX; + fB = fC; + } else if (4 <= fHPrime && fHPrime < 5) { + fR = fX; + fG = 0; + fB = fC; + } else if (5 <= fHPrime && fHPrime < 6) { + fR = fC; + fG = 0; + fB = fX; + } else { + fR = 0; + fG = 0; + fB = 0; + } + + fR += fM; + fG += fM; + fB += fM; + return {fR * 255.0f, fG * 255.0f, fB * 255.0f}; +} + +static xrt_vec3 +raycoord(struct ht_view *htv, struct xrt_vec3 model_out) +{ + cv::Mat in_px_coords(1, 1, CV_32FC2); + float *write_in; + write_in = in_px_coords.ptr(0); + write_in[0] = model_out.x; + write_in[1] = model_out.y; + cv::Mat out_ray(1, 1, CV_32FC2); + + cv::fisheye::undistortPoints(in_px_coords, out_ray, htv->cameraMatrix, htv->distortion); + + + float n_x = out_ray.at(0, 0); + float n_y = out_ray.at(0, 1); + + + struct xrt_vec3 n = {n_x, n_y, 1.0f}; + + cv::Matx33f R = htv->rotate_camera_to_stereo_camera; + + struct xrt_vec3 o = { + (n.x * R(0, 0)) + (n.y * R(0, 1)) + (n.z * R(0, 2)), + (n.x * R(1, 0)) + (n.y * R(1, 1)) + (n.z * R(1, 2)), + (n.x * R(2, 0)) + (n.y * R(2, 1)) + (n.z * R(2, 2)), + }; + + math_vec3_scalar_mul(1.0f / o.z, &o); + return o; +} + + +/*! + * Returns a 2x3 transform matrix that takes you back from the blackbarred image to the original image. + */ + +static cv::Matx23f +blackbar(cv::Mat &in, cv::Mat &out, xrt_size out_size) +{ +#if 1 + // Easy to think about, always right, but pretty slow: + // Get a matrix from the original to the scaled down / blackbar'd image, then get one that goes back. + // Then just warpAffine() it. + // Easy in programmer time - never have to worry about off by one, special cases. We can come back and optimize + // later. + + // Do the black bars need to be on top and bottom, or on left and right? + float scale_down_w = (float)out_size.w / (float)in.cols; // 128/1280 = 0.1 + float scale_down_h = (float)out_size.h / (float)in.rows; // 128/800 = 0.16 + + float scale_down = fmin(scale_down_w, scale_down_h); // 0.1 + + float width_inside = (float)in.cols * scale_down; + float height_inside = (float)in.rows * scale_down; + + float translate_x = (out_size.w - width_inside) / 2; // should be 0 for 1280x800 + float translate_y = (out_size.h - height_inside) / 2; // should be (1280-800)/2 = 240 + + cv::Matx23f go; + // clang-format off + go(0,0) = scale_down; go(0,1) = 0.0f; go(0,2) = translate_x; + go(1,0) = 0.0f; go(1,1) = scale_down; go(1,2) = translate_y; + // clang-format on + + cv::warpAffine(in, out, go, cv::Size(out_size.w, out_size.h)); + + cv::Matx23f ret; + + // clang-format off + ret(0,0) = 1.0f/scale_down; ret(0,1) = 0.0f; ret(0,2) = -translate_x/scale_down; + ret(1,0) = 0.0f; ret(1,1) = 1.0f/scale_down; ret(1,2) = -translate_y/scale_down; + // clang-format on + + return ret; +#else + // Fast, always wrong if the input isn't square. You'd end up using something like this, plus some + // copyMakeBorder if you want to optimize. + if (aspect_ratio_input == aspect_ratio_output) { + cv::resize(in, out, {out_size.w, out_size.h}); + cv::Matx23f ret; + float scale_from_out_to_in = (float)in.cols / (float)out_size.w; + // clang-format off + ret(0,0) = scale_from_out_to_in; ret(0,1) = 0.0f; ret(0,2) = 0.0f; + ret(1,0) = 0.0f; ret(1,1) = scale_from_out_to_in; ret(1,2) = 0.0f; + // clang-format on + cv::imshow("hi", out); + cv::waitKey(1); + return ret; + } + assert(!"Uh oh! Unimplemented!"); + return {}; +#endif +} + +/*! + * This is a template so that we can use xrt_vec3 or xrt_vec2. + * Please don't use this for anything other than xrt_vec3 or xrt_vec2! + */ + +template +T +transformVecBy2x3(T in, cv::Matx23f warp_back) +{ + T rr; + rr.x = (in.x * warp_back(0, 0)) + (in.y * warp_back(0, 1)) + warp_back(0, 2); + rr.y = (in.x * warp_back(1, 0)) + (in.y * warp_back(1, 1)) + warp_back(1, 2); + return rr; +} + +//! Draw some dots. Factors out some boilerplate. +static void +handDot(cv::Mat &mat, xrt_vec2 place, float radius, float hue, float intensity, int type) +{ + cv::circle(mat, {(int)place.x, (int)place.y}, radius, hsv2rgb(hue * 360.0f, intensity, intensity), type); +} + +static void +centerAndRotationFromJoints(struct ht_view *htv, + const xrt_vec2 *wrist, + const xrt_vec2 *index, + const xrt_vec2 *middle, + const xrt_vec2 *little, + xrt_vec2 *out_center, + xrt_vec2 *out_wrist_to_middle) +{ + // Close to what Mediapipe does, but slightly different - just uses the middle proximal instead of "estimating" + // it from the pinky and index. + // at the end of the day I should probably do that basis vector filtering thing to get a nicer middle metacarpal + // from 6 keypoints (not thumb proximal) OR SHOULD I. because distortion. hmm + + // Feel free to look at the way MP does it, you can see it's different. + // https://github.com/google/mediapipe/blob/master/mediapipe/modules/holistic_landmark/calculators/hand_detections_from_pose_to_rects_calculator.cc + + // struct xrt_vec2 hand_center = m_vec2_mul_scalar(middle, 0.5) + m_vec2_mul_scalar(index, 0.5*(2.0f/3.0f)) + + // m_vec2_mul_scalar(little, 0.5f*((1.0f/3.0f))); // Middle proximal, straight-up. + // U_LOG_E("%f %f %f %f %f %f %f %f ", wrist.x, wrist.y, index.x, index.y, middle.x, middle.y, little.x, + // little.y); + *out_center = m_vec2_lerp(*middle, m_vec2_lerp(*index, *little, 1.0f / 3.0f), 0.25f); + + *out_wrist_to_middle = *out_center - *wrist; +} + +static DetectionModelOutput +rotatedRectFromJoints(struct ht_view *htv, xrt_vec2 center, xrt_vec2 wrist_to_middle, DetectionModelOutput *out) +{ + float box_size = m_vec2_len(wrist_to_middle) * 2.0f * 1.73f; + + double rot = atan2(wrist_to_middle.x, wrist_to_middle.y) * (-180.0f / M_PI); + + out->rotation = rot; + out->size = box_size; + out->center = center; + + cv::RotatedRect rrect = + cv::RotatedRect(cv::Point2f(out->center.x, out->center.y), cv::Size2f(out->size, out->size), out->rotation); + + + cv::Point2f vertices[4]; + rrect.points(vertices); + if (htv->htd->debug_scribble && htv->htd->dynamic_config.scribble_bounding_box) { + for (int i = 0; i < 4; i++) { + cv::Scalar b = cv::Scalar(10, 30, 30); + if (i == 3) { + b = cv::Scalar(255, 255, 0); + } + cv::line(htv->debug_out_to_this, vertices[i], vertices[(i + 1) % 4], b, 2); + } + } + // topright is 0. bottomright is 1. bottomleft is 2. topleft is 3. + + cv::Point2f src_tri[3] = {vertices[3], vertices[2], vertices[1]}; // top-left, bottom-left, bottom-right + + cv::Point2f dest_tri[3] = {cv::Point2f(0, 0), cv::Point2f(0, 224), cv::Point2f(224, 224)}; + + out->warp_there = getAffineTransform(src_tri, dest_tri); + out->warp_back = getAffineTransform(dest_tri, src_tri); + + // out->wrist = wrist; + + return *out; +} + +static void +planarize(cv::Mat &input, uint8_t *output) +{ + // output better be the right size, because we are not doing any bounds checking! + assert(input.isContinuous()); + int lix = input.cols; + int liy = input.rows; + cv::Mat planes[3]; + cv::split(input, planes); + cv::Mat red = planes[0]; + cv::Mat green = planes[1]; + cv::Mat blue = planes[2]; + memcpy(output, red.data, lix * liy); + memcpy(output + (lix * liy), green.data, lix * liy); + memcpy(output + (lix * liy * 2), blue.data, lix * liy); +} diff --git a/src/xrt/drivers/ht/ht_interface.h b/src/xrt/drivers/ht/ht_interface.h index 4593a6b67..7a84c8d54 100644 --- a/src/xrt/drivers/ht/ht_interface.h +++ b/src/xrt/drivers/ht/ht_interface.h @@ -1,21 +1,38 @@ -// Copyright 2029, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Interface to camera based hand tracking driver code. * @author Christoph Haag + * @author Moses Turner * @ingroup drv_ht */ #pragma once -#include "math/m_api.h" #include "xrt/xrt_device.h" +#include "tracking/t_tracking.h" + + #ifdef __cplusplus extern "C" { #endif +enum ht_run_type +{ + HT_RUN_TYPE_VALVE_INDEX, + HT_RUN_TYPE_NORTH_STAR, +}; +// YES this is stupid. PLEASE bikeshed me on this when the time comes, this is terrible. + +// With Valve Index, we use the frameserver prober and look for the Valve Index camera, and we give the joint poses out +// in the space of the left (unrectified) camera. + +// With North Star, (really just Moses's headset :)) we hard-code to opening up a depthai_fs_stereo_rgb and give the +// joint poses out in the space of the "center" of the stereo camera. (Why? Because I don't have exact extrinsics from +// the NS "eyes" to the cameras. Less code this way.) + /*! * @defgroup drv_ht Camera based hand tracking * @ingroup drv @@ -24,15 +41,15 @@ extern "C" { */ /*! - * Create a probe for camera based hand tracking. + * Create a hand tracker device. * * @ingroup drv_ht */ -struct xrt_auto_prober * -ht_create_auto_prober(); +struct xrt_device * +ht_device_create(struct xrt_prober *xp, struct t_stereo_camera_calibration *calib); /*! - * @dir drivers/handtracking + * @dir drivers/ht * * @brief @ref drv_ht files. */ diff --git a/src/xrt/drivers/ht/ht_models.hpp b/src/xrt/drivers/ht/ht_models.hpp new file mode 100644 index 000000000..adec36d5e --- /dev/null +++ b/src/xrt/drivers/ht/ht_models.hpp @@ -0,0 +1,844 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Code to run machine learning models for camera-based hand tracker. + * @author Moses Turner + * @author Marcus Edel + * @ingroup drv_ht + */ + +// Many C api things were stolen from here (MIT license): +// https://github.com/microsoft/onnxruntime-inference-examples/blob/main/c_cxx/fns_candy_style_transfer/fns_candy_style_transfer.c + +#pragma once + +#include "os/os_time.h" +#include "os/os_threading.h" + +#include "math/m_api.h" +#include "math/m_vec2.h" +#include "math/m_vec3.h" + +#include "util/u_json.h" +#include "util/u_time.h" +#include "util/u_logging.h" +#include "util/u_trace_marker.h" + +#include "ht_nms.hpp" +#include "ht_driver.hpp" +#include "ht_image_math.hpp" + +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + + +#define ORT_CHECK(g_ort, expr) \ + do { \ + OrtStatus *onnx_status = (expr); \ + if (onnx_status != nullptr) { \ + const char *msg = g_ort->GetErrorMessage(onnx_status); \ + U_LOG_E("at %s:%d: %s\n", __FILE__, __LINE__, msg); \ + g_ort->ReleaseStatus(onnx_status); \ + assert(false); \ + } \ + } while (0); + +static std::vector> anchor{ + {0.031250, 0.031250, 1.000000, 1.000000}, {0.031250, 0.031250, 1.000000, 1.000000}, + {0.093750, 0.031250, 1.000000, 1.000000}, {0.093750, 0.031250, 1.000000, 1.000000}, + {0.156250, 0.031250, 1.000000, 1.000000}, {0.156250, 0.031250, 1.000000, 1.000000}, + {0.218750, 0.031250, 1.000000, 1.000000}, {0.218750, 0.031250, 1.000000, 1.000000}, + {0.281250, 0.031250, 1.000000, 1.000000}, {0.281250, 0.031250, 1.000000, 1.000000}, + {0.343750, 0.031250, 1.000000, 1.000000}, {0.343750, 0.031250, 1.000000, 1.000000}, + {0.406250, 0.031250, 1.000000, 1.000000}, {0.406250, 0.031250, 1.000000, 1.000000}, + {0.468750, 0.031250, 1.000000, 1.000000}, {0.468750, 0.031250, 1.000000, 1.000000}, + {0.531250, 0.031250, 1.000000, 1.000000}, {0.531250, 0.031250, 1.000000, 1.000000}, + {0.593750, 0.031250, 1.000000, 1.000000}, {0.593750, 0.031250, 1.000000, 1.000000}, + {0.656250, 0.031250, 1.000000, 1.000000}, {0.656250, 0.031250, 1.000000, 1.000000}, + {0.718750, 0.031250, 1.000000, 1.000000}, {0.718750, 0.031250, 1.000000, 1.000000}, + {0.781250, 0.031250, 1.000000, 1.000000}, {0.781250, 0.031250, 1.000000, 1.000000}, + {0.843750, 0.031250, 1.000000, 1.000000}, {0.843750, 0.031250, 1.000000, 1.000000}, + {0.906250, 0.031250, 1.000000, 1.000000}, {0.906250, 0.031250, 1.000000, 1.000000}, + {0.968750, 0.031250, 1.000000, 1.000000}, {0.968750, 0.031250, 1.000000, 1.000000}, + {0.031250, 0.093750, 1.000000, 1.000000}, {0.031250, 0.093750, 1.000000, 1.000000}, + {0.093750, 0.093750, 1.000000, 1.000000}, {0.093750, 0.093750, 1.000000, 1.000000}, + {0.156250, 0.093750, 1.000000, 1.000000}, {0.156250, 0.093750, 1.000000, 1.000000}, + {0.218750, 0.093750, 1.000000, 1.000000}, {0.218750, 0.093750, 1.000000, 1.000000}, + {0.281250, 0.093750, 1.000000, 1.000000}, {0.281250, 0.093750, 1.000000, 1.000000}, + {0.343750, 0.093750, 1.000000, 1.000000}, {0.343750, 0.093750, 1.000000, 1.000000}, + {0.406250, 0.093750, 1.000000, 1.000000}, {0.406250, 0.093750, 1.000000, 1.000000}, + {0.468750, 0.093750, 1.000000, 1.000000}, {0.468750, 0.093750, 1.000000, 1.000000}, + {0.531250, 0.093750, 1.000000, 1.000000}, {0.531250, 0.093750, 1.000000, 1.000000}, + {0.593750, 0.093750, 1.000000, 1.000000}, {0.593750, 0.093750, 1.000000, 1.000000}, + {0.656250, 0.093750, 1.000000, 1.000000}, {0.656250, 0.093750, 1.000000, 1.000000}, + {0.718750, 0.093750, 1.000000, 1.000000}, {0.718750, 0.093750, 1.000000, 1.000000}, + {0.781250, 0.093750, 1.000000, 1.000000}, {0.781250, 0.093750, 1.000000, 1.000000}, + {0.843750, 0.093750, 1.000000, 1.000000}, {0.843750, 0.093750, 1.000000, 1.000000}, + {0.906250, 0.093750, 1.000000, 1.000000}, {0.906250, 0.093750, 1.000000, 1.000000}, + {0.968750, 0.093750, 1.000000, 1.000000}, {0.968750, 0.093750, 1.000000, 1.000000}, + {0.031250, 0.156250, 1.000000, 1.000000}, {0.031250, 0.156250, 1.000000, 1.000000}, + {0.093750, 0.156250, 1.000000, 1.000000}, {0.093750, 0.156250, 1.000000, 1.000000}, + {0.156250, 0.156250, 1.000000, 1.000000}, {0.156250, 0.156250, 1.000000, 1.000000}, + {0.218750, 0.156250, 1.000000, 1.000000}, {0.218750, 0.156250, 1.000000, 1.000000}, + {0.281250, 0.156250, 1.000000, 1.000000}, {0.281250, 0.156250, 1.000000, 1.000000}, + {0.343750, 0.156250, 1.000000, 1.000000}, {0.343750, 0.156250, 1.000000, 1.000000}, + {0.406250, 0.156250, 1.000000, 1.000000}, {0.406250, 0.156250, 1.000000, 1.000000}, + {0.468750, 0.156250, 1.000000, 1.000000}, {0.468750, 0.156250, 1.000000, 1.000000}, + {0.531250, 0.156250, 1.000000, 1.000000}, {0.531250, 0.156250, 1.000000, 1.000000}, + {0.593750, 0.156250, 1.000000, 1.000000}, {0.593750, 0.156250, 1.000000, 1.000000}, + {0.656250, 0.156250, 1.000000, 1.000000}, {0.656250, 0.156250, 1.000000, 1.000000}, + {0.718750, 0.156250, 1.000000, 1.000000}, {0.718750, 0.156250, 1.000000, 1.000000}, + {0.781250, 0.156250, 1.000000, 1.000000}, {0.781250, 0.156250, 1.000000, 1.000000}, + {0.843750, 0.156250, 1.000000, 1.000000}, {0.843750, 0.156250, 1.000000, 1.000000}, + {0.906250, 0.156250, 1.000000, 1.000000}, {0.906250, 0.156250, 1.000000, 1.000000}, + {0.968750, 0.156250, 1.000000, 1.000000}, {0.968750, 0.156250, 1.000000, 1.000000}, + {0.031250, 0.218750, 1.000000, 1.000000}, {0.031250, 0.218750, 1.000000, 1.000000}, + {0.093750, 0.218750, 1.000000, 1.000000}, {0.093750, 0.218750, 1.000000, 1.000000}, + {0.156250, 0.218750, 1.000000, 1.000000}, {0.156250, 0.218750, 1.000000, 1.000000}, + {0.218750, 0.218750, 1.000000, 1.000000}, {0.218750, 0.218750, 1.000000, 1.000000}, + {0.281250, 0.218750, 1.000000, 1.000000}, {0.281250, 0.218750, 1.000000, 1.000000}, + {0.343750, 0.218750, 1.000000, 1.000000}, {0.343750, 0.218750, 1.000000, 1.000000}, + {0.406250, 0.218750, 1.000000, 1.000000}, {0.406250, 0.218750, 1.000000, 1.000000}, + {0.468750, 0.218750, 1.000000, 1.000000}, {0.468750, 0.218750, 1.000000, 1.000000}, + {0.531250, 0.218750, 1.000000, 1.000000}, {0.531250, 0.218750, 1.000000, 1.000000}, + {0.593750, 0.218750, 1.000000, 1.000000}, {0.593750, 0.218750, 1.000000, 1.000000}, + {0.656250, 0.218750, 1.000000, 1.000000}, {0.656250, 0.218750, 1.000000, 1.000000}, + {0.718750, 0.218750, 1.000000, 1.000000}, {0.718750, 0.218750, 1.000000, 1.000000}, + {0.781250, 0.218750, 1.000000, 1.000000}, {0.781250, 0.218750, 1.000000, 1.000000}, + {0.843750, 0.218750, 1.000000, 1.000000}, {0.843750, 0.218750, 1.000000, 1.000000}, + {0.906250, 0.218750, 1.000000, 1.000000}, {0.906250, 0.218750, 1.000000, 1.000000}, + {0.968750, 0.218750, 1.000000, 1.000000}, {0.968750, 0.218750, 1.000000, 1.000000}, + {0.031250, 0.281250, 1.000000, 1.000000}, {0.031250, 0.281250, 1.000000, 1.000000}, + {0.093750, 0.281250, 1.000000, 1.000000}, {0.093750, 0.281250, 1.000000, 1.000000}, + {0.156250, 0.281250, 1.000000, 1.000000}, {0.156250, 0.281250, 1.000000, 1.000000}, + {0.218750, 0.281250, 1.000000, 1.000000}, {0.218750, 0.281250, 1.000000, 1.000000}, + {0.281250, 0.281250, 1.000000, 1.000000}, {0.281250, 0.281250, 1.000000, 1.000000}, + {0.343750, 0.281250, 1.000000, 1.000000}, {0.343750, 0.281250, 1.000000, 1.000000}, + {0.406250, 0.281250, 1.000000, 1.000000}, {0.406250, 0.281250, 1.000000, 1.000000}, + {0.468750, 0.281250, 1.000000, 1.000000}, {0.468750, 0.281250, 1.000000, 1.000000}, + {0.531250, 0.281250, 1.000000, 1.000000}, {0.531250, 0.281250, 1.000000, 1.000000}, + {0.593750, 0.281250, 1.000000, 1.000000}, {0.593750, 0.281250, 1.000000, 1.000000}, + {0.656250, 0.281250, 1.000000, 1.000000}, {0.656250, 0.281250, 1.000000, 1.000000}, + {0.718750, 0.281250, 1.000000, 1.000000}, {0.718750, 0.281250, 1.000000, 1.000000}, + {0.781250, 0.281250, 1.000000, 1.000000}, {0.781250, 0.281250, 1.000000, 1.000000}, + {0.843750, 0.281250, 1.000000, 1.000000}, {0.843750, 0.281250, 1.000000, 1.000000}, + {0.906250, 0.281250, 1.000000, 1.000000}, {0.906250, 0.281250, 1.000000, 1.000000}, + {0.968750, 0.281250, 1.000000, 1.000000}, {0.968750, 0.281250, 1.000000, 1.000000}, + {0.031250, 0.343750, 1.000000, 1.000000}, {0.031250, 0.343750, 1.000000, 1.000000}, + {0.093750, 0.343750, 1.000000, 1.000000}, {0.093750, 0.343750, 1.000000, 1.000000}, + {0.156250, 0.343750, 1.000000, 1.000000}, {0.156250, 0.343750, 1.000000, 1.000000}, + {0.218750, 0.343750, 1.000000, 1.000000}, {0.218750, 0.343750, 1.000000, 1.000000}, + {0.281250, 0.343750, 1.000000, 1.000000}, {0.281250, 0.343750, 1.000000, 1.000000}, + {0.343750, 0.343750, 1.000000, 1.000000}, {0.343750, 0.343750, 1.000000, 1.000000}, + {0.406250, 0.343750, 1.000000, 1.000000}, {0.406250, 0.343750, 1.000000, 1.000000}, + {0.468750, 0.343750, 1.000000, 1.000000}, {0.468750, 0.343750, 1.000000, 1.000000}, + {0.531250, 0.343750, 1.000000, 1.000000}, {0.531250, 0.343750, 1.000000, 1.000000}, + {0.593750, 0.343750, 1.000000, 1.000000}, {0.593750, 0.343750, 1.000000, 1.000000}, + {0.656250, 0.343750, 1.000000, 1.000000}, {0.656250, 0.343750, 1.000000, 1.000000}, + {0.718750, 0.343750, 1.000000, 1.000000}, {0.718750, 0.343750, 1.000000, 1.000000}, + {0.781250, 0.343750, 1.000000, 1.000000}, {0.781250, 0.343750, 1.000000, 1.000000}, + {0.843750, 0.343750, 1.000000, 1.000000}, {0.843750, 0.343750, 1.000000, 1.000000}, + {0.906250, 0.343750, 1.000000, 1.000000}, {0.906250, 0.343750, 1.000000, 1.000000}, + {0.968750, 0.343750, 1.000000, 1.000000}, {0.968750, 0.343750, 1.000000, 1.000000}, + {0.031250, 0.406250, 1.000000, 1.000000}, {0.031250, 0.406250, 1.000000, 1.000000}, + {0.093750, 0.406250, 1.000000, 1.000000}, {0.093750, 0.406250, 1.000000, 1.000000}, + {0.156250, 0.406250, 1.000000, 1.000000}, {0.156250, 0.406250, 1.000000, 1.000000}, + {0.218750, 0.406250, 1.000000, 1.000000}, {0.218750, 0.406250, 1.000000, 1.000000}, + {0.281250, 0.406250, 1.000000, 1.000000}, {0.281250, 0.406250, 1.000000, 1.000000}, + {0.343750, 0.406250, 1.000000, 1.000000}, {0.343750, 0.406250, 1.000000, 1.000000}, + {0.406250, 0.406250, 1.000000, 1.000000}, {0.406250, 0.406250, 1.000000, 1.000000}, + {0.468750, 0.406250, 1.000000, 1.000000}, {0.468750, 0.406250, 1.000000, 1.000000}, + {0.531250, 0.406250, 1.000000, 1.000000}, {0.531250, 0.406250, 1.000000, 1.000000}, + {0.593750, 0.406250, 1.000000, 1.000000}, {0.593750, 0.406250, 1.000000, 1.000000}, + {0.656250, 0.406250, 1.000000, 1.000000}, {0.656250, 0.406250, 1.000000, 1.000000}, + {0.718750, 0.406250, 1.000000, 1.000000}, {0.718750, 0.406250, 1.000000, 1.000000}, + {0.781250, 0.406250, 1.000000, 1.000000}, {0.781250, 0.406250, 1.000000, 1.000000}, + {0.843750, 0.406250, 1.000000, 1.000000}, {0.843750, 0.406250, 1.000000, 1.000000}, + {0.906250, 0.406250, 1.000000, 1.000000}, {0.906250, 0.406250, 1.000000, 1.000000}, + {0.968750, 0.406250, 1.000000, 1.000000}, {0.968750, 0.406250, 1.000000, 1.000000}, + {0.031250, 0.468750, 1.000000, 1.000000}, {0.031250, 0.468750, 1.000000, 1.000000}, + {0.093750, 0.468750, 1.000000, 1.000000}, {0.093750, 0.468750, 1.000000, 1.000000}, + {0.156250, 0.468750, 1.000000, 1.000000}, {0.156250, 0.468750, 1.000000, 1.000000}, + {0.218750, 0.468750, 1.000000, 1.000000}, {0.218750, 0.468750, 1.000000, 1.000000}, + {0.281250, 0.468750, 1.000000, 1.000000}, {0.281250, 0.468750, 1.000000, 1.000000}, + {0.343750, 0.468750, 1.000000, 1.000000}, {0.343750, 0.468750, 1.000000, 1.000000}, + {0.406250, 0.468750, 1.000000, 1.000000}, {0.406250, 0.468750, 1.000000, 1.000000}, + {0.468750, 0.468750, 1.000000, 1.000000}, {0.468750, 0.468750, 1.000000, 1.000000}, + {0.531250, 0.468750, 1.000000, 1.000000}, {0.531250, 0.468750, 1.000000, 1.000000}, + {0.593750, 0.468750, 1.000000, 1.000000}, {0.593750, 0.468750, 1.000000, 1.000000}, + {0.656250, 0.468750, 1.000000, 1.000000}, {0.656250, 0.468750, 1.000000, 1.000000}, + {0.718750, 0.468750, 1.000000, 1.000000}, {0.718750, 0.468750, 1.000000, 1.000000}, + {0.781250, 0.468750, 1.000000, 1.000000}, {0.781250, 0.468750, 1.000000, 1.000000}, + {0.843750, 0.468750, 1.000000, 1.000000}, {0.843750, 0.468750, 1.000000, 1.000000}, + {0.906250, 0.468750, 1.000000, 1.000000}, {0.906250, 0.468750, 1.000000, 1.000000}, + {0.968750, 0.468750, 1.000000, 1.000000}, {0.968750, 0.468750, 1.000000, 1.000000}, + {0.031250, 0.531250, 1.000000, 1.000000}, {0.031250, 0.531250, 1.000000, 1.000000}, + {0.093750, 0.531250, 1.000000, 1.000000}, {0.093750, 0.531250, 1.000000, 1.000000}, + {0.156250, 0.531250, 1.000000, 1.000000}, {0.156250, 0.531250, 1.000000, 1.000000}, + {0.218750, 0.531250, 1.000000, 1.000000}, {0.218750, 0.531250, 1.000000, 1.000000}, + {0.281250, 0.531250, 1.000000, 1.000000}, {0.281250, 0.531250, 1.000000, 1.000000}, + {0.343750, 0.531250, 1.000000, 1.000000}, {0.343750, 0.531250, 1.000000, 1.000000}, + {0.406250, 0.531250, 1.000000, 1.000000}, {0.406250, 0.531250, 1.000000, 1.000000}, + {0.468750, 0.531250, 1.000000, 1.000000}, {0.468750, 0.531250, 1.000000, 1.000000}, + {0.531250, 0.531250, 1.000000, 1.000000}, {0.531250, 0.531250, 1.000000, 1.000000}, + {0.593750, 0.531250, 1.000000, 1.000000}, {0.593750, 0.531250, 1.000000, 1.000000}, + {0.656250, 0.531250, 1.000000, 1.000000}, {0.656250, 0.531250, 1.000000, 1.000000}, + {0.718750, 0.531250, 1.000000, 1.000000}, {0.718750, 0.531250, 1.000000, 1.000000}, + {0.781250, 0.531250, 1.000000, 1.000000}, {0.781250, 0.531250, 1.000000, 1.000000}, + {0.843750, 0.531250, 1.000000, 1.000000}, {0.843750, 0.531250, 1.000000, 1.000000}, + {0.906250, 0.531250, 1.000000, 1.000000}, {0.906250, 0.531250, 1.000000, 1.000000}, + {0.968750, 0.531250, 1.000000, 1.000000}, {0.968750, 0.531250, 1.000000, 1.000000}, + {0.031250, 0.593750, 1.000000, 1.000000}, {0.031250, 0.593750, 1.000000, 1.000000}, + {0.093750, 0.593750, 1.000000, 1.000000}, {0.093750, 0.593750, 1.000000, 1.000000}, + {0.156250, 0.593750, 1.000000, 1.000000}, {0.156250, 0.593750, 1.000000, 1.000000}, + {0.218750, 0.593750, 1.000000, 1.000000}, {0.218750, 0.593750, 1.000000, 1.000000}, + {0.281250, 0.593750, 1.000000, 1.000000}, {0.281250, 0.593750, 1.000000, 1.000000}, + {0.343750, 0.593750, 1.000000, 1.000000}, {0.343750, 0.593750, 1.000000, 1.000000}, + {0.406250, 0.593750, 1.000000, 1.000000}, {0.406250, 0.593750, 1.000000, 1.000000}, + {0.468750, 0.593750, 1.000000, 1.000000}, {0.468750, 0.593750, 1.000000, 1.000000}, + {0.531250, 0.593750, 1.000000, 1.000000}, {0.531250, 0.593750, 1.000000, 1.000000}, + {0.593750, 0.593750, 1.000000, 1.000000}, {0.593750, 0.593750, 1.000000, 1.000000}, + {0.656250, 0.593750, 1.000000, 1.000000}, {0.656250, 0.593750, 1.000000, 1.000000}, + {0.718750, 0.593750, 1.000000, 1.000000}, {0.718750, 0.593750, 1.000000, 1.000000}, + {0.781250, 0.593750, 1.000000, 1.000000}, {0.781250, 0.593750, 1.000000, 1.000000}, + {0.843750, 0.593750, 1.000000, 1.000000}, {0.843750, 0.593750, 1.000000, 1.000000}, + {0.906250, 0.593750, 1.000000, 1.000000}, {0.906250, 0.593750, 1.000000, 1.000000}, + {0.968750, 0.593750, 1.000000, 1.000000}, {0.968750, 0.593750, 1.000000, 1.000000}, + {0.031250, 0.656250, 1.000000, 1.000000}, {0.031250, 0.656250, 1.000000, 1.000000}, + {0.093750, 0.656250, 1.000000, 1.000000}, {0.093750, 0.656250, 1.000000, 1.000000}, + {0.156250, 0.656250, 1.000000, 1.000000}, {0.156250, 0.656250, 1.000000, 1.000000}, + {0.218750, 0.656250, 1.000000, 1.000000}, {0.218750, 0.656250, 1.000000, 1.000000}, + {0.281250, 0.656250, 1.000000, 1.000000}, {0.281250, 0.656250, 1.000000, 1.000000}, + {0.343750, 0.656250, 1.000000, 1.000000}, {0.343750, 0.656250, 1.000000, 1.000000}, + {0.406250, 0.656250, 1.000000, 1.000000}, {0.406250, 0.656250, 1.000000, 1.000000}, + {0.468750, 0.656250, 1.000000, 1.000000}, {0.468750, 0.656250, 1.000000, 1.000000}, + {0.531250, 0.656250, 1.000000, 1.000000}, {0.531250, 0.656250, 1.000000, 1.000000}, + {0.593750, 0.656250, 1.000000, 1.000000}, {0.593750, 0.656250, 1.000000, 1.000000}, + {0.656250, 0.656250, 1.000000, 1.000000}, {0.656250, 0.656250, 1.000000, 1.000000}, + {0.718750, 0.656250, 1.000000, 1.000000}, {0.718750, 0.656250, 1.000000, 1.000000}, + {0.781250, 0.656250, 1.000000, 1.000000}, {0.781250, 0.656250, 1.000000, 1.000000}, + {0.843750, 0.656250, 1.000000, 1.000000}, {0.843750, 0.656250, 1.000000, 1.000000}, + {0.906250, 0.656250, 1.000000, 1.000000}, {0.906250, 0.656250, 1.000000, 1.000000}, + {0.968750, 0.656250, 1.000000, 1.000000}, {0.968750, 0.656250, 1.000000, 1.000000}, + {0.031250, 0.718750, 1.000000, 1.000000}, {0.031250, 0.718750, 1.000000, 1.000000}, + {0.093750, 0.718750, 1.000000, 1.000000}, {0.093750, 0.718750, 1.000000, 1.000000}, + {0.156250, 0.718750, 1.000000, 1.000000}, {0.156250, 0.718750, 1.000000, 1.000000}, + {0.218750, 0.718750, 1.000000, 1.000000}, {0.218750, 0.718750, 1.000000, 1.000000}, + {0.281250, 0.718750, 1.000000, 1.000000}, {0.281250, 0.718750, 1.000000, 1.000000}, + {0.343750, 0.718750, 1.000000, 1.000000}, {0.343750, 0.718750, 1.000000, 1.000000}, + {0.406250, 0.718750, 1.000000, 1.000000}, {0.406250, 0.718750, 1.000000, 1.000000}, + {0.468750, 0.718750, 1.000000, 1.000000}, {0.468750, 0.718750, 1.000000, 1.000000}, + {0.531250, 0.718750, 1.000000, 1.000000}, {0.531250, 0.718750, 1.000000, 1.000000}, + {0.593750, 0.718750, 1.000000, 1.000000}, {0.593750, 0.718750, 1.000000, 1.000000}, + {0.656250, 0.718750, 1.000000, 1.000000}, {0.656250, 0.718750, 1.000000, 1.000000}, + {0.718750, 0.718750, 1.000000, 1.000000}, {0.718750, 0.718750, 1.000000, 1.000000}, + {0.781250, 0.718750, 1.000000, 1.000000}, {0.781250, 0.718750, 1.000000, 1.000000}, + {0.843750, 0.718750, 1.000000, 1.000000}, {0.843750, 0.718750, 1.000000, 1.000000}, + {0.906250, 0.718750, 1.000000, 1.000000}, {0.906250, 0.718750, 1.000000, 1.000000}, + {0.968750, 0.718750, 1.000000, 1.000000}, {0.968750, 0.718750, 1.000000, 1.000000}, + {0.031250, 0.781250, 1.000000, 1.000000}, {0.031250, 0.781250, 1.000000, 1.000000}, + {0.093750, 0.781250, 1.000000, 1.000000}, {0.093750, 0.781250, 1.000000, 1.000000}, + {0.156250, 0.781250, 1.000000, 1.000000}, {0.156250, 0.781250, 1.000000, 1.000000}, + {0.218750, 0.781250, 1.000000, 1.000000}, {0.218750, 0.781250, 1.000000, 1.000000}, + {0.281250, 0.781250, 1.000000, 1.000000}, {0.281250, 0.781250, 1.000000, 1.000000}, + {0.343750, 0.781250, 1.000000, 1.000000}, {0.343750, 0.781250, 1.000000, 1.000000}, + {0.406250, 0.781250, 1.000000, 1.000000}, {0.406250, 0.781250, 1.000000, 1.000000}, + {0.468750, 0.781250, 1.000000, 1.000000}, {0.468750, 0.781250, 1.000000, 1.000000}, + {0.531250, 0.781250, 1.000000, 1.000000}, {0.531250, 0.781250, 1.000000, 1.000000}, + {0.593750, 0.781250, 1.000000, 1.000000}, {0.593750, 0.781250, 1.000000, 1.000000}, + {0.656250, 0.781250, 1.000000, 1.000000}, {0.656250, 0.781250, 1.000000, 1.000000}, + {0.718750, 0.781250, 1.000000, 1.000000}, {0.718750, 0.781250, 1.000000, 1.000000}, + {0.781250, 0.781250, 1.000000, 1.000000}, {0.781250, 0.781250, 1.000000, 1.000000}, + {0.843750, 0.781250, 1.000000, 1.000000}, {0.843750, 0.781250, 1.000000, 1.000000}, + {0.906250, 0.781250, 1.000000, 1.000000}, {0.906250, 0.781250, 1.000000, 1.000000}, + {0.968750, 0.781250, 1.000000, 1.000000}, {0.968750, 0.781250, 1.000000, 1.000000}, + {0.031250, 0.843750, 1.000000, 1.000000}, {0.031250, 0.843750, 1.000000, 1.000000}, + {0.093750, 0.843750, 1.000000, 1.000000}, {0.093750, 0.843750, 1.000000, 1.000000}, + {0.156250, 0.843750, 1.000000, 1.000000}, {0.156250, 0.843750, 1.000000, 1.000000}, + {0.218750, 0.843750, 1.000000, 1.000000}, {0.218750, 0.843750, 1.000000, 1.000000}, + {0.281250, 0.843750, 1.000000, 1.000000}, {0.281250, 0.843750, 1.000000, 1.000000}, + {0.343750, 0.843750, 1.000000, 1.000000}, {0.343750, 0.843750, 1.000000, 1.000000}, + {0.406250, 0.843750, 1.000000, 1.000000}, {0.406250, 0.843750, 1.000000, 1.000000}, + {0.468750, 0.843750, 1.000000, 1.000000}, {0.468750, 0.843750, 1.000000, 1.000000}, + {0.531250, 0.843750, 1.000000, 1.000000}, {0.531250, 0.843750, 1.000000, 1.000000}, + {0.593750, 0.843750, 1.000000, 1.000000}, {0.593750, 0.843750, 1.000000, 1.000000}, + {0.656250, 0.843750, 1.000000, 1.000000}, {0.656250, 0.843750, 1.000000, 1.000000}, + {0.718750, 0.843750, 1.000000, 1.000000}, {0.718750, 0.843750, 1.000000, 1.000000}, + {0.781250, 0.843750, 1.000000, 1.000000}, {0.781250, 0.843750, 1.000000, 1.000000}, + {0.843750, 0.843750, 1.000000, 1.000000}, {0.843750, 0.843750, 1.000000, 1.000000}, + {0.906250, 0.843750, 1.000000, 1.000000}, {0.906250, 0.843750, 1.000000, 1.000000}, + {0.968750, 0.843750, 1.000000, 1.000000}, {0.968750, 0.843750, 1.000000, 1.000000}, + {0.031250, 0.906250, 1.000000, 1.000000}, {0.031250, 0.906250, 1.000000, 1.000000}, + {0.093750, 0.906250, 1.000000, 1.000000}, {0.093750, 0.906250, 1.000000, 1.000000}, + {0.156250, 0.906250, 1.000000, 1.000000}, {0.156250, 0.906250, 1.000000, 1.000000}, + {0.218750, 0.906250, 1.000000, 1.000000}, {0.218750, 0.906250, 1.000000, 1.000000}, + {0.281250, 0.906250, 1.000000, 1.000000}, {0.281250, 0.906250, 1.000000, 1.000000}, + {0.343750, 0.906250, 1.000000, 1.000000}, {0.343750, 0.906250, 1.000000, 1.000000}, + {0.406250, 0.906250, 1.000000, 1.000000}, {0.406250, 0.906250, 1.000000, 1.000000}, + {0.468750, 0.906250, 1.000000, 1.000000}, {0.468750, 0.906250, 1.000000, 1.000000}, + {0.531250, 0.906250, 1.000000, 1.000000}, {0.531250, 0.906250, 1.000000, 1.000000}, + {0.593750, 0.906250, 1.000000, 1.000000}, {0.593750, 0.906250, 1.000000, 1.000000}, + {0.656250, 0.906250, 1.000000, 1.000000}, {0.656250, 0.906250, 1.000000, 1.000000}, + {0.718750, 0.906250, 1.000000, 1.000000}, {0.718750, 0.906250, 1.000000, 1.000000}, + {0.781250, 0.906250, 1.000000, 1.000000}, {0.781250, 0.906250, 1.000000, 1.000000}, + {0.843750, 0.906250, 1.000000, 1.000000}, {0.843750, 0.906250, 1.000000, 1.000000}, + {0.906250, 0.906250, 1.000000, 1.000000}, {0.906250, 0.906250, 1.000000, 1.000000}, + {0.968750, 0.906250, 1.000000, 1.000000}, {0.968750, 0.906250, 1.000000, 1.000000}, + {0.031250, 0.968750, 1.000000, 1.000000}, {0.031250, 0.968750, 1.000000, 1.000000}, + {0.093750, 0.968750, 1.000000, 1.000000}, {0.093750, 0.968750, 1.000000, 1.000000}, + {0.156250, 0.968750, 1.000000, 1.000000}, {0.156250, 0.968750, 1.000000, 1.000000}, + {0.218750, 0.968750, 1.000000, 1.000000}, {0.218750, 0.968750, 1.000000, 1.000000}, + {0.281250, 0.968750, 1.000000, 1.000000}, {0.281250, 0.968750, 1.000000, 1.000000}, + {0.343750, 0.968750, 1.000000, 1.000000}, {0.343750, 0.968750, 1.000000, 1.000000}, + {0.406250, 0.968750, 1.000000, 1.000000}, {0.406250, 0.968750, 1.000000, 1.000000}, + {0.468750, 0.968750, 1.000000, 1.000000}, {0.468750, 0.968750, 1.000000, 1.000000}, + {0.531250, 0.968750, 1.000000, 1.000000}, {0.531250, 0.968750, 1.000000, 1.000000}, + {0.593750, 0.968750, 1.000000, 1.000000}, {0.593750, 0.968750, 1.000000, 1.000000}, + {0.656250, 0.968750, 1.000000, 1.000000}, {0.656250, 0.968750, 1.000000, 1.000000}, + {0.718750, 0.968750, 1.000000, 1.000000}, {0.718750, 0.968750, 1.000000, 1.000000}, + {0.781250, 0.968750, 1.000000, 1.000000}, {0.781250, 0.968750, 1.000000, 1.000000}, + {0.843750, 0.968750, 1.000000, 1.000000}, {0.843750, 0.968750, 1.000000, 1.000000}, + {0.906250, 0.968750, 1.000000, 1.000000}, {0.906250, 0.968750, 1.000000, 1.000000}, + {0.968750, 0.968750, 1.000000, 1.000000}, {0.968750, 0.968750, 1.000000, 1.000000}, + {0.062500, 0.062500, 1.000000, 1.000000}, {0.062500, 0.062500, 1.000000, 1.000000}, + {0.187500, 0.062500, 1.000000, 1.000000}, {0.187500, 0.062500, 1.000000, 1.000000}, + {0.312500, 0.062500, 1.000000, 1.000000}, {0.312500, 0.062500, 1.000000, 1.000000}, + {0.437500, 0.062500, 1.000000, 1.000000}, {0.437500, 0.062500, 1.000000, 1.000000}, + {0.562500, 0.062500, 1.000000, 1.000000}, {0.562500, 0.062500, 1.000000, 1.000000}, + {0.687500, 0.062500, 1.000000, 1.000000}, {0.687500, 0.062500, 1.000000, 1.000000}, + {0.812500, 0.062500, 1.000000, 1.000000}, {0.812500, 0.062500, 1.000000, 1.000000}, + {0.937500, 0.062500, 1.000000, 1.000000}, {0.937500, 0.062500, 1.000000, 1.000000}, + {0.062500, 0.187500, 1.000000, 1.000000}, {0.062500, 0.187500, 1.000000, 1.000000}, + {0.187500, 0.187500, 1.000000, 1.000000}, {0.187500, 0.187500, 1.000000, 1.000000}, + {0.312500, 0.187500, 1.000000, 1.000000}, {0.312500, 0.187500, 1.000000, 1.000000}, + {0.437500, 0.187500, 1.000000, 1.000000}, {0.437500, 0.187500, 1.000000, 1.000000}, + {0.562500, 0.187500, 1.000000, 1.000000}, {0.562500, 0.187500, 1.000000, 1.000000}, + {0.687500, 0.187500, 1.000000, 1.000000}, {0.687500, 0.187500, 1.000000, 1.000000}, + {0.812500, 0.187500, 1.000000, 1.000000}, {0.812500, 0.187500, 1.000000, 1.000000}, + {0.937500, 0.187500, 1.000000, 1.000000}, {0.937500, 0.187500, 1.000000, 1.000000}, + {0.062500, 0.312500, 1.000000, 1.000000}, {0.062500, 0.312500, 1.000000, 1.000000}, + {0.187500, 0.312500, 1.000000, 1.000000}, {0.187500, 0.312500, 1.000000, 1.000000}, + {0.312500, 0.312500, 1.000000, 1.000000}, {0.312500, 0.312500, 1.000000, 1.000000}, + {0.437500, 0.312500, 1.000000, 1.000000}, {0.437500, 0.312500, 1.000000, 1.000000}, + {0.562500, 0.312500, 1.000000, 1.000000}, {0.562500, 0.312500, 1.000000, 1.000000}, + {0.687500, 0.312500, 1.000000, 1.000000}, {0.687500, 0.312500, 1.000000, 1.000000}, + {0.812500, 0.312500, 1.000000, 1.000000}, {0.812500, 0.312500, 1.000000, 1.000000}, + {0.937500, 0.312500, 1.000000, 1.000000}, {0.937500, 0.312500, 1.000000, 1.000000}, + {0.062500, 0.437500, 1.000000, 1.000000}, {0.062500, 0.437500, 1.000000, 1.000000}, + {0.187500, 0.437500, 1.000000, 1.000000}, {0.187500, 0.437500, 1.000000, 1.000000}, + {0.312500, 0.437500, 1.000000, 1.000000}, {0.312500, 0.437500, 1.000000, 1.000000}, + {0.437500, 0.437500, 1.000000, 1.000000}, {0.437500, 0.437500, 1.000000, 1.000000}, + {0.562500, 0.437500, 1.000000, 1.000000}, {0.562500, 0.437500, 1.000000, 1.000000}, + {0.687500, 0.437500, 1.000000, 1.000000}, {0.687500, 0.437500, 1.000000, 1.000000}, + {0.812500, 0.437500, 1.000000, 1.000000}, {0.812500, 0.437500, 1.000000, 1.000000}, + {0.937500, 0.437500, 1.000000, 1.000000}, {0.937500, 0.437500, 1.000000, 1.000000}, + {0.062500, 0.562500, 1.000000, 1.000000}, {0.062500, 0.562500, 1.000000, 1.000000}, + {0.187500, 0.562500, 1.000000, 1.000000}, {0.187500, 0.562500, 1.000000, 1.000000}, + {0.312500, 0.562500, 1.000000, 1.000000}, {0.312500, 0.562500, 1.000000, 1.000000}, + {0.437500, 0.562500, 1.000000, 1.000000}, {0.437500, 0.562500, 1.000000, 1.000000}, + {0.562500, 0.562500, 1.000000, 1.000000}, {0.562500, 0.562500, 1.000000, 1.000000}, + {0.687500, 0.562500, 1.000000, 1.000000}, {0.687500, 0.562500, 1.000000, 1.000000}, + {0.812500, 0.562500, 1.000000, 1.000000}, {0.812500, 0.562500, 1.000000, 1.000000}, + {0.937500, 0.562500, 1.000000, 1.000000}, {0.937500, 0.562500, 1.000000, 1.000000}, + {0.062500, 0.687500, 1.000000, 1.000000}, {0.062500, 0.687500, 1.000000, 1.000000}, + {0.187500, 0.687500, 1.000000, 1.000000}, {0.187500, 0.687500, 1.000000, 1.000000}, + {0.312500, 0.687500, 1.000000, 1.000000}, {0.312500, 0.687500, 1.000000, 1.000000}, + {0.437500, 0.687500, 1.000000, 1.000000}, {0.437500, 0.687500, 1.000000, 1.000000}, + {0.562500, 0.687500, 1.000000, 1.000000}, {0.562500, 0.687500, 1.000000, 1.000000}, + {0.687500, 0.687500, 1.000000, 1.000000}, {0.687500, 0.687500, 1.000000, 1.000000}, + {0.812500, 0.687500, 1.000000, 1.000000}, {0.812500, 0.687500, 1.000000, 1.000000}, + {0.937500, 0.687500, 1.000000, 1.000000}, {0.937500, 0.687500, 1.000000, 1.000000}, + {0.062500, 0.812500, 1.000000, 1.000000}, {0.062500, 0.812500, 1.000000, 1.000000}, + {0.187500, 0.812500, 1.000000, 1.000000}, {0.187500, 0.812500, 1.000000, 1.000000}, + {0.312500, 0.812500, 1.000000, 1.000000}, {0.312500, 0.812500, 1.000000, 1.000000}, + {0.437500, 0.812500, 1.000000, 1.000000}, {0.437500, 0.812500, 1.000000, 1.000000}, + {0.562500, 0.812500, 1.000000, 1.000000}, {0.562500, 0.812500, 1.000000, 1.000000}, + {0.687500, 0.812500, 1.000000, 1.000000}, {0.687500, 0.812500, 1.000000, 1.000000}, + {0.812500, 0.812500, 1.000000, 1.000000}, {0.812500, 0.812500, 1.000000, 1.000000}, + {0.937500, 0.812500, 1.000000, 1.000000}, {0.937500, 0.812500, 1.000000, 1.000000}, + {0.062500, 0.937500, 1.000000, 1.000000}, {0.062500, 0.937500, 1.000000, 1.000000}, + {0.187500, 0.937500, 1.000000, 1.000000}, {0.187500, 0.937500, 1.000000, 1.000000}, + {0.312500, 0.937500, 1.000000, 1.000000}, {0.312500, 0.937500, 1.000000, 1.000000}, + {0.437500, 0.937500, 1.000000, 1.000000}, {0.437500, 0.937500, 1.000000, 1.000000}, + {0.562500, 0.937500, 1.000000, 1.000000}, {0.562500, 0.937500, 1.000000, 1.000000}, + {0.687500, 0.937500, 1.000000, 1.000000}, {0.687500, 0.937500, 1.000000, 1.000000}, + {0.812500, 0.937500, 1.000000, 1.000000}, {0.812500, 0.937500, 1.000000, 1.000000}, + {0.937500, 0.937500, 1.000000, 1.000000}, {0.937500, 0.937500, 1.000000, 1.000000}, + {0.125000, 0.125000, 1.000000, 1.000000}, {0.125000, 0.125000, 1.000000, 1.000000}, + {0.125000, 0.125000, 1.000000, 1.000000}, {0.125000, 0.125000, 1.000000, 1.000000}, + {0.125000, 0.125000, 1.000000, 1.000000}, {0.125000, 0.125000, 1.000000, 1.000000}, + {0.375000, 0.125000, 1.000000, 1.000000}, {0.375000, 0.125000, 1.000000, 1.000000}, + {0.375000, 0.125000, 1.000000, 1.000000}, {0.375000, 0.125000, 1.000000, 1.000000}, + {0.375000, 0.125000, 1.000000, 1.000000}, {0.375000, 0.125000, 1.000000, 1.000000}, + {0.625000, 0.125000, 1.000000, 1.000000}, {0.625000, 0.125000, 1.000000, 1.000000}, + {0.625000, 0.125000, 1.000000, 1.000000}, {0.625000, 0.125000, 1.000000, 1.000000}, + {0.625000, 0.125000, 1.000000, 1.000000}, {0.625000, 0.125000, 1.000000, 1.000000}, + {0.875000, 0.125000, 1.000000, 1.000000}, {0.875000, 0.125000, 1.000000, 1.000000}, + {0.875000, 0.125000, 1.000000, 1.000000}, {0.875000, 0.125000, 1.000000, 1.000000}, + {0.875000, 0.125000, 1.000000, 1.000000}, {0.875000, 0.125000, 1.000000, 1.000000}, + {0.125000, 0.375000, 1.000000, 1.000000}, {0.125000, 0.375000, 1.000000, 1.000000}, + {0.125000, 0.375000, 1.000000, 1.000000}, {0.125000, 0.375000, 1.000000, 1.000000}, + {0.125000, 0.375000, 1.000000, 1.000000}, {0.125000, 0.375000, 1.000000, 1.000000}, + {0.375000, 0.375000, 1.000000, 1.000000}, {0.375000, 0.375000, 1.000000, 1.000000}, + {0.375000, 0.375000, 1.000000, 1.000000}, {0.375000, 0.375000, 1.000000, 1.000000}, + {0.375000, 0.375000, 1.000000, 1.000000}, {0.375000, 0.375000, 1.000000, 1.000000}, + {0.625000, 0.375000, 1.000000, 1.000000}, {0.625000, 0.375000, 1.000000, 1.000000}, + {0.625000, 0.375000, 1.000000, 1.000000}, {0.625000, 0.375000, 1.000000, 1.000000}, + {0.625000, 0.375000, 1.000000, 1.000000}, {0.625000, 0.375000, 1.000000, 1.000000}, + {0.875000, 0.375000, 1.000000, 1.000000}, {0.875000, 0.375000, 1.000000, 1.000000}, + {0.875000, 0.375000, 1.000000, 1.000000}, {0.875000, 0.375000, 1.000000, 1.000000}, + {0.875000, 0.375000, 1.000000, 1.000000}, {0.875000, 0.375000, 1.000000, 1.000000}, + {0.125000, 0.625000, 1.000000, 1.000000}, {0.125000, 0.625000, 1.000000, 1.000000}, + {0.125000, 0.625000, 1.000000, 1.000000}, {0.125000, 0.625000, 1.000000, 1.000000}, + {0.125000, 0.625000, 1.000000, 1.000000}, {0.125000, 0.625000, 1.000000, 1.000000}, + {0.375000, 0.625000, 1.000000, 1.000000}, {0.375000, 0.625000, 1.000000, 1.000000}, + {0.375000, 0.625000, 1.000000, 1.000000}, {0.375000, 0.625000, 1.000000, 1.000000}, + {0.375000, 0.625000, 1.000000, 1.000000}, {0.375000, 0.625000, 1.000000, 1.000000}, + {0.625000, 0.625000, 1.000000, 1.000000}, {0.625000, 0.625000, 1.000000, 1.000000}, + {0.625000, 0.625000, 1.000000, 1.000000}, {0.625000, 0.625000, 1.000000, 1.000000}, + {0.625000, 0.625000, 1.000000, 1.000000}, {0.625000, 0.625000, 1.000000, 1.000000}, + {0.875000, 0.625000, 1.000000, 1.000000}, {0.875000, 0.625000, 1.000000, 1.000000}, + {0.875000, 0.625000, 1.000000, 1.000000}, {0.875000, 0.625000, 1.000000, 1.000000}, + {0.875000, 0.625000, 1.000000, 1.000000}, {0.875000, 0.625000, 1.000000, 1.000000}, + {0.125000, 0.875000, 1.000000, 1.000000}, {0.125000, 0.875000, 1.000000, 1.000000}, + {0.125000, 0.875000, 1.000000, 1.000000}, {0.125000, 0.875000, 1.000000, 1.000000}, + {0.125000, 0.875000, 1.000000, 1.000000}, {0.125000, 0.875000, 1.000000, 1.000000}, + {0.375000, 0.875000, 1.000000, 1.000000}, {0.375000, 0.875000, 1.000000, 1.000000}, + {0.375000, 0.875000, 1.000000, 1.000000}, {0.375000, 0.875000, 1.000000, 1.000000}, + {0.375000, 0.875000, 1.000000, 1.000000}, {0.375000, 0.875000, 1.000000, 1.000000}, + {0.625000, 0.875000, 1.000000, 1.000000}, {0.625000, 0.875000, 1.000000, 1.000000}, + {0.625000, 0.875000, 1.000000, 1.000000}, {0.625000, 0.875000, 1.000000, 1.000000}, + {0.625000, 0.875000, 1.000000, 1.000000}, {0.625000, 0.875000, 1.000000, 1.000000}, + {0.875000, 0.875000, 1.000000, 1.000000}, {0.875000, 0.875000, 1.000000, 1.000000}, + {0.875000, 0.875000, 1.000000, 1.000000}, {0.875000, 0.875000, 1.000000, 1.000000}, + {0.875000, 0.875000, 1.000000, 1.000000}, {0.875000, 0.875000, 1.000000, 1.000000}}; + +static Hand2D +runKeypointEstimator(struct ht_view *htv, cv::Mat img) +{ + constexpr size_t lix = 224; + constexpr size_t liy = 224; + constexpr size_t nb_planes = 3; + cv::Mat planes[nb_planes]; + + constexpr size_t size = lix * liy * nb_planes; + + std::vector combined_planes(size); + planarize(img, combined_planes.data()); + + // Normalize - supposedly, the keypoint estimator wants keypoints in [0,1] + std::vector real_thing(size); + for (size_t i = 0; i < size; i++) { + real_thing[i] = (float)combined_planes[i] / 255.0; + } + + const OrtApi *g_ort = htv->htd->ort_api; + struct ModelInfo *model = &htv->keypoint_model; + + OrtValue *input_tensor = nullptr; + + ORT_CHECK(g_ort, g_ort->CreateTensorWithDataAsOrtValue( + model->memoryInfo, real_thing.data(), model->input_size_bytes, model->input_shape.data(), + model->input_shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor)); + + // Cargo-culted + assert(input_tensor != nullptr); + int is_tensor; + ORT_CHECK(g_ort, g_ort->IsTensor(input_tensor, &is_tensor)); + assert(is_tensor); + + const char *output_names[] = {"Identity", "Identity_2", "Identity_2"}; + + // If any of these are non-NULL, ONNX will explode and won't tell you why! Be extremely paranoid! + OrtValue *output_tensor[3] = {nullptr, nullptr, nullptr}; + ORT_CHECK(g_ort, g_ort->Run(model->session, nullptr, model->input_names.data(), &input_tensor, 1, output_names, + 3, output_tensor)); + + ORT_CHECK(g_ort, g_ort->IsTensor(output_tensor[0], &is_tensor)); + assert(is_tensor); + + float *landmarks = nullptr; + + // Should give a pointer to data that is freed on g_ort->ReleaseValue(output_tensor[0]);. + ORT_CHECK(g_ort, g_ort->GetTensorMutableData(output_tensor[0], (void **)&landmarks)); + + int stride = 3; + Hand2D dumb; + for (size_t i = 0; i < 21; i++) { + int rt = i * stride; + float x = landmarks[rt]; + float y = landmarks[rt + 1]; + float z = landmarks[rt + 2]; + dumb.kps[i].x = x; + dumb.kps[i].y = y; + dumb.kps[i].z = z; + } + + // We have to get to here, or else we leak a whole lot! If you need to return early, make this into a goto! + g_ort->ReleaseValue(output_tensor[0]); + g_ort->ReleaseValue(output_tensor[1]); + g_ort->ReleaseValue(output_tensor[2]); + g_ort->ReleaseValue(input_tensor); + return dumb; +} + +#undef HEAVY_SCRIBBLE + + +static std::vector +runHandDetector(struct ht_view *htv, cv::Mat &raw_input) +{ + const OrtApi *g_ort = htv->htd->ort_api; + + cv::Mat img; + + constexpr int hd_size = 128; + constexpr size_t nb_planes = 3; + constexpr size_t size = hd_size * hd_size * nb_planes; + constexpr int inputTensorSize = size * sizeof(float); + + cv::Matx23f back_from_blackbar = blackbar(raw_input, img, {hd_size, hd_size}); + + float scale_factor = back_from_blackbar(0, 0); // 960/128 + assert(img.isContinuous()); + constexpr float mean = 128.0f; + constexpr float std = 128.0f; + + std::vector real_thing(size); + + if (htv->htd->startup_config.palm_detection_use_mediapipe) { + std::vector combined_planes(size); + planarize(img, combined_planes.data()); + for (size_t i = 0; i < size; i++) { + float val = (float)combined_planes[i]; + real_thing[i] = (val - mean) / std; + } + } else { + + assert(img.isContinuous()); + + for (size_t i = 0; i < size; i++) { + int val = img.data[i]; + + real_thing[i] = (val - mean) / std; + } + } + + // Convenience + struct ModelInfo *model_hd = &htv->detection_model; + + // If this is non-NULL, ONNX will explode and won't tell you why! + OrtValue *input_tensor = nullptr; + + ORT_CHECK(g_ort, g_ort->CreateTensorWithDataAsOrtValue( + model_hd->memoryInfo, real_thing.data(), inputTensorSize, model_hd->input_shape.data(), + model_hd->input_shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor)); + + // Cargo-culted + assert(input_tensor != nullptr); + int is_tensor; + ORT_CHECK(g_ort, g_ort->IsTensor(input_tensor, &is_tensor)); + assert(is_tensor); + + // If any of these are non-NULL, ONNX will explode and won't tell you why! Be extremely paranoid! + OrtValue *output_tensor[2] = {nullptr, nullptr}; + ORT_CHECK(g_ort, g_ort->Run(model_hd->session, nullptr, model_hd->input_names.data(), + (const OrtValue *const *)&input_tensor, model_hd->input_names.size(), + model_hd->output_names.data(), model_hd->output_names.size(), output_tensor)); + + ORT_CHECK(g_ort, g_ort->IsTensor(output_tensor[0], &is_tensor)); + assert(is_tensor); + + float *classificators = nullptr; + float *regressors = nullptr; + + ORT_CHECK(g_ort, + g_ort->GetTensorMutableData(output_tensor[0], (void **)&classificators)); // Totally won't segfault! + ORT_CHECK(g_ort, g_ort->GetTensorMutableData(output_tensor[1], (void **)®ressors)); + // We need to free these! + + float *rg = regressors; // shorter name + + std::vector detections; + int count = 0; + size_t i = 0; + + std::vector output; + std::vector nms_palms; + + + for (std::vector>::iterator it = anchor.begin(); it != anchor.end(); ++it, ++i) { + std::vector::iterator anchorData = it->begin(); + + float score0 = classificators[i]; + float score = 1.0 / (1.0 + exp(-score0)); + + // Let a lot of detections in - they'll be slowly rejected later + if (score > htv->htd->dynamic_config.nms_threshold.val) { + // Boundary box. + NMSPalm det; + + float anchx = *(anchorData + 0) * 128; + float anchy = *(anchorData + 1) * 128; + + float shiftx = regressors[i * 18]; + float shifty = regressors[i * 18 + 1]; + + float w = regressors[i * 18 + 2]; + float h = regressors[i * 18 + 3]; + + + + float cx = shiftx + anchx; + float cy = shifty + anchy; + + struct xrt_vec2 *kps = det.keypoints; + + kps[0] = {rg[i * 18 + 4], rg[i * 18 + 5]}; + kps[1] = {rg[i * 18 + 6], rg[i * 18 + 7]}; + kps[2] = {rg[i * 18 + 8], rg[i * 18 + 9]}; + kps[3] = {rg[i * 18 + 10], rg[i * 18 + 11]}; + kps[4] = {rg[i * 18 + 12], rg[i * 18 + 13]}; + kps[5] = {rg[i * 18 + 14], rg[i * 18 + 15]}; + kps[6] = {rg[i * 18 + 16], rg[i * 18 + 17]}; + + + for (int i = 0; i < 7; i++) { + struct xrt_vec2 *b = &kps[i]; + b->x += anchx; + b->y += anchy; + } + + det.bbox.w = w; + det.bbox.h = h; + det.bbox.cx = cx; + det.bbox.cy = cy; + det.confidence = score; + detections.push_back(det); + count++; + + if (htv->htd->debug_scribble && (htv->htd->dynamic_config.scribble_raw_detections)) { + xrt_vec2 center = transformVecBy2x3(xrt_vec2{cx, cy}, back_from_blackbar); + + float sz = det.bbox.w * scale_factor; + + cv::rectangle( + htv->debug_out_to_this, + {(int)(center.x - (sz / 2)), (int)(center.y - (sz / 2)), (int)sz, (int)sz}, + hsv2rgb(0.0f, math_map_ranges(det.confidence, 0.0f, 1.0f, 1.5f, -0.1f), + math_map_ranges(det.confidence, 0.0f, 1.0f, 0.2f, 1.4f)), + 1); + + for (int i = 0; i < 7; i++) { + handDot(htv->debug_out_to_this, transformVecBy2x3(kps[i], back_from_blackbar), + det.confidence * 7, ((float)i) * (360.0f / 7.0f), det.confidence, 1); + } + } + + + + int square = fmax(w, h); + + square = square / scale_factor; + } + } + + if (count == 0) { + goto cleanup; + } + + nms_palms = filterBoxesWeightedAvg(detections, htv->htd->dynamic_config.nms_iou.val); + + + + for (NMSPalm cooler : nms_palms) { + + // Display box + + struct xrt_vec2 tl = {cooler.bbox.cx - cooler.bbox.w / 2, cooler.bbox.cy - cooler.bbox.h / 2}; + struct xrt_vec2 bob = transformVecBy2x3(tl, back_from_blackbar); + float sz = cooler.bbox.w * scale_factor; + + if (htv->htd->debug_scribble && htv->htd->dynamic_config.scribble_nms_detections) { + cv::rectangle(htv->debug_out_to_this, {(int)bob.x, (int)bob.y, (int)sz, (int)sz}, + hsv2rgb(180.0f, math_map_ranges(cooler.confidence, 0.0f, 1.0f, 0.8f, -0.1f), + math_map_ranges(cooler.confidence, 0.0f, 1.0f, 0.2f, 1.4f)), + 2); + for (int i = 0; i < 7; i++) { + handDot(htv->debug_out_to_this, + transformVecBy2x3(cooler.keypoints[i], back_from_blackbar), + cooler.confidence * 14, ((float)i) * (360.0f / 7.0f), cooler.confidence, 3); + } + } + + + Palm7KP this_element; + + for (int i = 0; i < 7; i++) { + struct xrt_vec2 b = cooler.keypoints[i]; + this_element.kps[i] = transformVecBy2x3(b, back_from_blackbar); + } + this_element.confidence = cooler.confidence; + + output.push_back(this_element); + } + +cleanup: + g_ort->ReleaseValue(output_tensor[0]); + g_ort->ReleaseValue(output_tensor[1]); + g_ort->ReleaseValue(input_tensor); + return output; +} + + +static void +addSlug(struct ht_device *htd, const char *suffix, char *out) +{ + strcpy(out, htd->startup_config.model_slug); + strcat(out, suffix); +} + +static void +initKeypointEstimator(struct ht_device *htd, ht_view *htv) +{ + struct ModelInfo *model_ke = &htv->keypoint_model; + const OrtApi *g_ort = htd->ort_api; + OrtSessionOptions *opts = nullptr; + + ORT_CHECK(g_ort, g_ort->CreateSessionOptions(&opts)); + + ORT_CHECK(g_ort, g_ort->SetSessionGraphOptimizationLevel(opts, ORT_ENABLE_ALL)); + ORT_CHECK(g_ort, g_ort->SetIntraOpNumThreads(opts, 1)); + + char modelLocation[1024]; + if (htd->startup_config.keypoint_estimation_use_mediapipe) { + addSlug(htd, "hand_landmark_MEDIAPIPE.onnx", modelLocation); + } else { + addSlug(htd, "hand_landmark_COLLABORA.onnx", modelLocation); + } + ORT_CHECK(g_ort, g_ort->CreateSession(htd->ort_env, modelLocation, opts, &model_ke->session)); + g_ort->ReleaseSessionOptions(opts); + + model_ke->input_shape.push_back(1); + model_ke->input_shape.push_back(3); + model_ke->input_shape.push_back(224); + model_ke->input_shape.push_back(224); + + model_ke->input_names.push_back("input_1"); + + model_ke->output_names.push_back("Identity"); + model_ke->output_names.push_back("Identity_1"); + model_ke->output_names.push_back("Identity_2"); + + model_ke->input_size_bytes = 224 * 224 * 3 * sizeof(float); + + ORT_CHECK(g_ort, g_ort->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &model_ke->memoryInfo)); + + htv->run_keypoint_model = runKeypointEstimator; +} + +static void +initHandDetector(struct ht_device *htd, ht_view *htv) +{ + struct ModelInfo *model_hd = &htv->detection_model; + memset(model_hd, 0, sizeof(struct ModelInfo)); + + const OrtApi *g_ort = htd->ort_api; + OrtSessionOptions *opts = nullptr; + ORT_CHECK(g_ort, g_ort->CreateSessionOptions(&opts)); + + ORT_CHECK(g_ort, g_ort->SetSessionGraphOptimizationLevel(opts, ORT_ENABLE_ALL)); + ORT_CHECK(g_ort, g_ort->SetIntraOpNumThreads(opts, 1)); + + char modelLocation[1024]; + + // Hard-coded. Even though you can use the ONNX runtime's API to dynamically figure these out, that doesn't make + // any sense because these don't change between runs, and if you are swapping models you have to do much more + // than just change the input/output names. + if (htd->startup_config.palm_detection_use_mediapipe) { + addSlug(htd, "palm_detection_MEDIAPIPE.onnx", modelLocation); + model_hd->input_shape.push_back(1); + model_hd->input_shape.push_back(3); + model_hd->input_shape.push_back(128); + model_hd->input_shape.push_back(128); + + model_hd->input_names.push_back("input"); + } else { + addSlug(htd, "palm_detection_COLLABORA.onnx", modelLocation); + + model_hd->input_shape.push_back(1); + model_hd->input_shape.push_back(128); + model_hd->input_shape.push_back(128); + model_hd->input_shape.push_back(3); + + model_hd->input_names.push_back("input:0"); + } + + ORT_CHECK(g_ort, g_ort->CreateSession(htd->ort_env, modelLocation, opts, &model_hd->session)); + g_ort->ReleaseSessionOptions(opts); + + + + model_hd->output_names.push_back("classificators"); + model_hd->output_names.push_back("regressors"); + + ORT_CHECK(g_ort, g_ort->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &model_hd->memoryInfo)); + + htv->run_detection_model = runHandDetector; +} + + +static void +initOnnx(struct ht_device *htd) +{ + htd->ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION); + ORT_CHECK(htd->ort_api, htd->ort_api->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "moses", &htd->ort_env)); + + initHandDetector(htd, &htd->views[0]); + initHandDetector(htd, &htd->views[1]); + + initKeypointEstimator(htd, &htd->views[0]); + initKeypointEstimator(htd, &htd->views[1]); +} + +static void +destroyModelInfo(struct ht_device *htd, ModelInfo *info) +{ + const OrtApi *g_ort = htd->ort_api; + g_ort->ReleaseSession(info->session); + g_ort->ReleaseMemoryInfo(info->memoryInfo); + // Same deal as in ht_device - I'm mixing C and C++ idioms, so sometimes it's easier to just manually call their + // destructors instead of figuring out some way to convince C++ to call them implicitly. + info->output_names.~vector(); + info->input_names.~vector(); + info->input_shape.~vector(); +} + +void +destroyOnnx(struct ht_device *htd) +{ + destroyModelInfo(htd, &htd->views[0].keypoint_model); + destroyModelInfo(htd, &htd->views[1].keypoint_model); + destroyModelInfo(htd, &htd->views[0].detection_model); + destroyModelInfo(htd, &htd->views[1].detection_model); + htd->ort_api->ReleaseEnv(htd->ort_env); +} diff --git a/src/xrt/drivers/ht/ht_nms.hpp b/src/xrt/drivers/ht/ht_nms.hpp new file mode 100644 index 000000000..2888d9098 --- /dev/null +++ b/src/xrt/drivers/ht/ht_nms.hpp @@ -0,0 +1,164 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Code to deal with bounding boxes for camera-based hand-tracking. + * @author Moses Turner + * @author Marcus Edel + * @ingroup drv_ht + */ + +#pragma once + +#include "xrt/xrt_defines.h" +#include "ht_driver.hpp" + +#include +#include + +#include + + +struct Box +{ + float cx; + float cy; + float w; + float h; +}; + +struct NMSPalm +{ + Box bbox; + xrt_vec2 keypoints[7]; + float confidence; +}; + +static float +overlap(float x1, float w1, float x2, float w2) +{ + float l1 = x1 - w1 / 2; + float l2 = x2 - w2 / 2; + float left = l1 > l2 ? l1 : l2; + + float r1 = x1 + w1 / 2; + float r2 = x2 + w2 / 2; + float right = r1 < r2 ? r1 : r2; + + return right - left; +} + +static float +boxIntersection(const Box &a, const Box &b) +{ + float w = overlap(a.cx, a.w, b.cx, b.w); + float h = overlap(a.cy, a.h, b.cy, b.h); + + if (w < 0 || h < 0) + return 0; + + return w * h; +} + +static float +boxUnion(const Box &a, const Box &b) +{ + return a.w * a.h + b.w * b.h - boxIntersection(a, b); +} + +static float +boxIOU(const Box &a, const Box &b) +{ + return boxIntersection(a, b) / boxUnion(a, b); +} + +static NMSPalm +weightedAvgBoxes(std::vector &detections) +{ + float weight = 0.0f; // or, sum_confidences. + float cx = 0.0f; + float cy = 0.0f; + float size = 0.0f; + NMSPalm out = {}; + + for (NMSPalm &detection : detections) { + weight += detection.confidence; + cx += detection.bbox.cx * detection.confidence; + cy += detection.bbox.cy * detection.confidence; + size += detection.bbox.w * .5 * detection.confidence; + size += detection.bbox.h * .5 * detection.confidence; + + for (int i = 0; i < 7; i++) { + out.keypoints[i].x += detection.keypoints[i].x * detection.confidence; + out.keypoints[i].y += detection.keypoints[i].y * detection.confidence; + } + } + cx /= weight; + cy /= weight; + size /= weight; + for (int i = 0; i < 7; i++) { + out.keypoints[i].x /= weight; + out.keypoints[i].y /= weight; + } + + + float bare_confidence = weight / detections.size(); + + // desmos \frac{1}{1+e^{-.5x}}-.5 + + float steep = 0.2; + float cent = 0.5; + + float exp = detections.size(); + + float sigmoid_addendum = (1.0f / (1.0f + pow(M_E, (-steep * exp)))) - cent; + + float diff_bare_to_one = 1.0f - bare_confidence; + + out.confidence = bare_confidence + (sigmoid_addendum * diff_bare_to_one); + + // U_LOG_E("Bare %f num %f sig %f diff %f out %f", bare_confidence, exp, sigmoid_addendum, diff_bare_to_one, + // out.confidence); + + out.bbox.cx = cx; + out.bbox.cy = cy; + out.bbox.w = size; + out.bbox.h = size; + return out; +} + +static std::vector +filterBoxesWeightedAvg(std::vector &detections, float min_iou = 0.1f) +{ + std::vector> overlaps; + std::vector outs; + + + // U_LOG_D("\n\nStarting filtering boxes. There are %zu boxes to look at.\n", detections.size()); + for (NMSPalm &detection : detections) { + // U_LOG_D("Starting looking at one detection\n"); + bool foundAHome = false; + for (size_t i = 0; i < outs.size(); i++) { + float iou = boxIOU(outs[i].bbox, detection.bbox); + // U_LOG_D("IOU is %f\n", iou); + // U_LOG_D("Outs box is %f %f %f %f", outs[i].bbox.cx, outs[i].bbox.cy, outs[i].bbox.w, + // outs[i].bbox.h) + if (iou > min_iou) { + // This one intersects with the whole thing + overlaps[i].push_back(detection); + outs[i] = weightedAvgBoxes(overlaps[i]); + foundAHome = true; + break; + } + } + if (!foundAHome) { + // U_LOG_D("No home\n"); + overlaps.push_back({detection}); + outs.push_back({detection}); + } else { + // U_LOG_D("Found a home!\n"); + } + } + // U_LOG_D("Sizeeeeeeeeeeeeeeeeeeeee is %zu\n", outs.size()); + return outs; +} diff --git a/src/xrt/drivers/ht/ht_prober.c b/src/xrt/drivers/ht/ht_prober.c deleted file mode 100644 index 7165a1883..000000000 --- a/src/xrt/drivers/ht/ht_prober.c +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2020, Collabora, Ltd. -// SPDX-License-Identifier: BSL-1.0 -/*! - * @file - * @brief Camera based hand tracking prober code. - * @author Christoph Haag - * @ingroup drv_ht - */ - - -#include "xrt/xrt_prober.h" - -#include "util/u_misc.h" - -#include "ht_interface.h" -#include "ht_driver.h" - -/*! - * @implements xrt_auto_prober - */ -struct ht_prober -{ - struct xrt_auto_prober base; -}; - -//! @private @memberof ht_prober -static inline struct ht_prober * -ht_prober(struct xrt_auto_prober *p) -{ - return (struct ht_prober *)p; -} - -//! @public @memberof ht_prober -static void -ht_prober_destroy(struct xrt_auto_prober *p) -{ - struct ht_prober *htp = ht_prober(p); - - free(htp); -} - -//! @public @memberof ht_prober -static struct xrt_device * -ht_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no_hmds, struct xrt_prober *xp) -{ - struct xrt_device *xdev = ht_device_create(xap, attached_data, xp); - - if (xdev == NULL) { - return NULL; - } - - xdev->orientation_tracking_supported = true; - xdev->position_tracking_supported = true; - xdev->hand_tracking_supported = true; - xdev->device_type = XRT_DEVICE_TYPE_HAND_TRACKER; - - return xdev; -} - -struct xrt_auto_prober * -ht_create_auto_prober() -{ - struct ht_prober *htp = U_TYPED_CALLOC(struct ht_prober); - htp->base.name = "Camera Hand Tracking"; - htp->base.destroy = ht_prober_destroy; - htp->base.lelo_dallas_autoprobe = ht_prober_autoprobe; - - return &htp->base; -} diff --git a/src/xrt/drivers/ht/readme.md b/src/xrt/drivers/ht/readme.md new file mode 100644 index 000000000..b77cb8cab --- /dev/null +++ b/src/xrt/drivers/ht/readme.md @@ -0,0 +1,90 @@ + + +# What is this? +This is a driver to do optical hand tracking. The actual code mostly written by Moses Turner, with tons of help from Marcus Edel, Jakob Bornecrantz, Ryan Pavlik, and Christoph Haag. Jakob Bornecrantz and Marcus Edel are the main people who gathered training data for the initial Collabora models. + +Currently, it works with the Valve Index. In the past, it was tested with a Luxonis 1090ffc, and in the future it should work fine with devices like the T265, Leap Motion Controller (w/ LeapUVC), or PS4/PS5 cam, should there be enough interest for any of those. + +Under good lighting, I would say it's around as good as Oculus Quest 2's hand tracking. Not that I'm trying to make any claims; that's just what I honestly would tell somebody if they are wondering if it's worth testing out. + + +# How to get started +## Get dependencies +### Get OpenCV +Each distro has its own way to get OpenCV, and it can change at any time; there's no specific reason to trust this documentation over anything else. + +Having said that, on Ubuntu, it would look something like + +``` +sudo apt install libopencv-dev libopencv-contrib-dev +``` + +Or you could build it from source, or get it from one of the other 1000s of package managers. Whatever floats your boat. + +### Get ONNXRuntime +I followed the instructions here: https://onnxruntime.ai/docs/how-to/build/inferencing.html#linux + +then had to do +``` +cd build/Linux/RelWithDebInfo/ +sudo make install +``` + +### Get the ML models +Make sure you have git-lfs installed, then run ./scripts/get-ht-models.sh. Should work fine. + +## Building the driver +Once onnxruntime is installed, you should be able to build like normal with CMake or Meson. + +If it properly found everything, - CMake should say + +``` +-- Found ONNXRUNTIME: /usr/local/include/onnxruntime + +[...] + +-- # DRIVER_HANDTRACKING: ON +``` + +and Meson should say + +``` +Run-time dependency libonnxruntime found: YES 1.8.2 + +[...] + +Message: Configuration done! +Message: drivers: [...] handtracking, [...] +``` + +## Running the driver +Currently, it's only set up to work on Valve Index. + +So, the two things you can do are +* Use the `survive` driver with both controllers off - It should automagically start hand tracking upon not finding any controllers. +* Use the `vive` driver with `VIVE_USE_HANDTRACKING=ON` and it should work the same as the survive driver. + +You can see if the driver is working with `openxr-simple-playground`, StereoKit, or any other app you know of. Poke me (Moses) if you find any other cool hand-tracking apps; I'm always looking for more! + +# Tips and tricks + +This tracking likes to be in a bright, evenly-lit room with multiple light sources. Turn all the lights on, see if you can find any lamps. If the ML models can see well, the tracking quality can get surprisingly nice. + +Sometimes, the tracking fails when it can see more than one hand. As the tracking gets better (we train better ML models and squash more bugs) this should happen less often or not at all. If it does, put one of your hands down, and it should resume tracking the remaining hand just fine. + +# Future improvements + +* Get more training data; train better ML models. +* Improve the tracking math + * Be smarter about keeping tracking lock on a hand + * Try predicting the next bounding box based on the estimated keypoints of the last few frames instead of blindly trusting the detection model, and not run the detection model *every single* frame. + * Instead of directly doing disparity on the observed keypoints, use a kinematic model of the hand and fit that to the 2D observations - this should get rid of a *lot* of jitter and make it look better to the end user if the ML models fail + * Make something that also works with non-stereo (mono, trinocular, or N cameras) camera setups +* Optionally run the ML models on GPU - currently, everything's CPU bound which could be dumb under some circumstances +* Write a lot of generic code so that you can run this on any stereo camera +* More advanced prediction/interpolation code that doesn't care at all about the input frame cadence. One-euro filters are pretty good about this, but we can get better! \ No newline at end of file diff --git a/src/xrt/drivers/ht/templates/NaivePermutationSort.hpp b/src/xrt/drivers/ht/templates/NaivePermutationSort.hpp new file mode 100644 index 000000000..c8cdc9d7c --- /dev/null +++ b/src/xrt/drivers/ht/templates/NaivePermutationSort.hpp @@ -0,0 +1,91 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Camera based hand tracking sorting implementation. + * @author Moses Turner + * @ingroup drv_ht + */ + +#pragma once + +#include +#include +#include +#include +// Other thing: sort by speed? like, if our thing must have suddenly changed directions, add to error? +// Easy enough to do using more complicated structs. +// Like a past thing with position, velocity and timestamp - present thing with position and timestamp. + +// typedef bool booool; + +struct psort_atom_t +{ + size_t idx_1; + size_t idx_2; + float err; +}; + + +bool +comp_err(psort_atom_t one, psort_atom_t two) +{ + return (one.err < two.err); +} + + +template +void +naive_sort_permutation_by_error( + // Inputs - shall be initialized with real data before calling. This function shall not modify them in any way. + std::vector &in_1, + std::vector &in_2, + + // Outputs - shall be uninitialized. This function shall initialize them to the right size and fill them with the + // proper values. + std::vector &used_1, + std::vector &used_2, + std::vector &out_indices_1, + std::vector &out_indices_2, + std::vector &out_errs, + + float (*calc_error)(Tp_1 *one, Tp_2 *two), + float max_err = std::numeric_limits::max()) +{ + used_1 = std::vector(in_1.size()); // silly? Unsure. + used_2 = std::vector(in_2.size()); + + size_t out_size = std::min(in_1.size(), in_2.size()); + + out_indices_1.reserve(out_size); + out_indices_2.reserve(out_size); + + std::vector associations; + + for (size_t idx_1 = 0; idx_1 < in_1.size(); idx_1++) { + for (size_t idx_2 = 0; idx_2 < in_2.size(); idx_2++) { + float err = calc_error(&in_1[idx_1], &in_2[idx_2]); + if (err > 0.0f) { + // Negative error means the error calculator thought there was something so bad with + // these that they shouldn't be considered at all. + associations.push_back({idx_1, idx_2, err}); + } + } + } + + std::sort(associations.begin(), associations.end(), comp_err); + + for (size_t i = 0; i < associations.size(); i++) { + psort_atom_t chonk = associations[i]; + if (used_1[chonk.idx_1] || used_2[chonk.idx_2] || (chonk.err > max_err)) { + continue; + } + used_1[chonk.idx_1] = true; + used_2[chonk.idx_2] = true; + + out_indices_1.push_back(chonk.idx_1); + out_indices_2.push_back(chonk.idx_2); + + out_errs.push_back(chonk.err); + } +} diff --git a/src/xrt/drivers/hydra/hydra_driver.c b/src/xrt/drivers/hydra/hydra_driver.c index 40d3d7b2d..174d82bdf 100644 --- a/src/xrt/drivers/hydra/hydra_driver.c +++ b/src/xrt/drivers/hydra/hydra_driver.c @@ -625,6 +625,7 @@ hydra_found(struct xrt_prober *xp, // hs->base.set_output = hydra_device_set_output; hd->base.name = XRT_DEVICE_HYDRA; snprintf(hd->base.str, XRT_DEVICE_NAME_LEN, "%s %i", "Razer Hydra Controller", (int)(i + 1)); + snprintf(hd->base.serial, XRT_DEVICE_NAME_LEN, "%s %i", "Razer Hydra Controller", (int)(i + 1)); SET_INPUT(1_CLICK); SET_INPUT(2_CLICK); SET_INPUT(3_CLICK); diff --git a/src/xrt/drivers/illixr/illixr_component.cpp b/src/xrt/drivers/illixr/illixr_component.cpp index 7b412c585..d802678f9 100644 --- a/src/xrt/drivers/illixr/illixr_component.cpp +++ b/src/xrt/drivers/illixr/illixr_component.cpp @@ -24,18 +24,11 @@ class illixr_plugin : public plugin { public: illixr_plugin(std::string name_, phonebook *pb_) - : plugin{name_, pb_}, sb{pb->lookup_impl()}, sb_pose{pb->lookup_impl()}, - sb_eyebuffer{sb->publish("eyebuffer")}, sb_vsync_estimate{sb->subscribe_latest( - "vsync_estimate")} + : plugin{name_, pb_}, sb{pb->lookup_impl()}, sb_pose{pb->lookup_impl()} {} const std::shared_ptr sb; const std::shared_ptr sb_pose; - const std::unique_ptr> sb_eyebuffer; - const std::unique_ptr> sb_vsync_estimate; - fast_pose_type prev_pose; /* stores a copy of pose each time - illixr_read_pose() is called */ - std::chrono::time_point sample_time; /* when prev_pose was stored */ }; static illixr_plugin *illixr_plugin_obj = nullptr; @@ -60,9 +53,6 @@ illixr_read_pose() const fast_pose_type fast_pose = illixr_plugin_obj->sb_pose->get_fast_pose(); const pose_type pose = fast_pose.pose; - // record when the pose was read for use in write_frame - illixr_plugin_obj->sample_time = std::chrono::system_clock::now(); - ret.orientation.x = pose.orientation.x(); ret.orientation.y = pose.orientation.y(); ret.orientation.z = pose.orientation.z(); @@ -71,8 +61,5 @@ illixr_read_pose() ret.position.y = pose.position.y(); ret.position.z = pose.position.z(); - // store pose in static variable for use in write_frame - illixr_plugin_obj->prev_pose = fast_pose; // copy member variables - return ret; -} \ No newline at end of file +} diff --git a/src/xrt/drivers/illixr/illixr_device.cpp b/src/xrt/drivers/illixr/illixr_device.cpp index 81b22f8ac..158a0aa96 100644 --- a/src/xrt/drivers/illixr/illixr_device.cpp +++ b/src/xrt/drivers/illixr/illixr_device.cpp @@ -132,13 +132,12 @@ illixr_hmd_get_tracked_pose(struct xrt_device *xdev, static void illixr_hmd_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { - struct xrt_pose pose = illixr_read_pose(); - - *out_pose = pose; + (void)xdev; + u_device_get_view_pose(eye_relation, view_index, out_pose); } std::vector @@ -177,7 +176,11 @@ illixr_hmd_create(const char *path_in, const char *comp_in) dh->base.destroy = illixr_hmd_destroy; dh->base.name = XRT_DEVICE_GENERIC_HMD; dh->base.device_type = XRT_DEVICE_TYPE_HMD; - dh->base.hmd->blend_mode = XRT_BLEND_MODE_OPAQUE; + + size_t idx = 0; + dh->base.hmd->blend_modes[idx++] = XRT_BLEND_MODE_OPAQUE; + dh->base.hmd->num_blend_modes = idx; + dh->pose.orientation.w = 1.0f; // All other values set to zero. dh->print_spew = debug_get_bool_option_illixr_spew(); dh->print_debug = debug_get_bool_option_illixr_debug(); @@ -186,6 +189,7 @@ illixr_hmd_create(const char *path_in, const char *comp_in) // Print name. snprintf(dh->base.str, XRT_DEVICE_NAME_LEN, "ILLIXR"); + snprintf(dh->base.serial, XRT_DEVICE_NAME_LEN, "ILLIXR"); // Setup input. dh->base.inputs[0].name = XRT_INPUT_GENERIC_HEAD_POSE; diff --git a/src/xrt/drivers/illixr/illixr_prober.c b/src/xrt/drivers/illixr/illixr_prober.c index 5dde4b93e..964bb8c95 100644 --- a/src/xrt/drivers/illixr/illixr_prober.c +++ b/src/xrt/drivers/illixr/illixr_prober.c @@ -33,30 +33,36 @@ illixr_prober_destroy(struct xrt_auto_prober *p) free(dp); } -static struct xrt_device * -illixr_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no_hmds, struct xrt_prober *xp) +static int +illixr_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) { struct illixr_prober *dp = illixr_prober(xap); (void)dp; if (no_hmds) { - return NULL; + return 0; } const char *illixr_path, *illixr_comp; illixr_path = getenv("ILLIXR_PATH"); illixr_comp = getenv("ILLIXR_COMP"); if (!illixr_path || !illixr_comp) { - return NULL; + return 0; } - return illixr_hmd_create(illixr_path, illixr_comp); + out_xdevs[0] = illixr_hmd_create(illixr_path, illixr_comp); + return 1; } struct xrt_auto_prober * illixr_create_auto_prober() { struct illixr_prober *dp = U_TYPED_CALLOC(struct illixr_prober); + dp->base.name = "ILLIXR"; dp->base.destroy = illixr_prober_destroy; dp->base.lelo_dallas_autoprobe = illixr_prober_autoprobe; diff --git a/src/xrt/drivers/meson.build b/src/xrt/drivers/meson.build index 848b9faf1..edd9b82fe 100644 --- a/src/xrt/drivers/meson.build +++ b/src/xrt/drivers/meson.build @@ -3,6 +3,21 @@ drv_include = include_directories('.') +lib_drv_depthai = static_library( + 'drv_depthai', + files( + 'depthai/depthai_driver.cpp', + ), + include_directories: [xrt_include], + dependencies: [aux, opencv, depthai], + build_by_default: 'depthai' in drivers, +) + +drv_depthai = declare_dependency( + include_directories: drv_include, + link_with: lib_drv_depthai +) + lib_drv_dummy = static_library( 'drv_dummy', files( @@ -42,9 +57,9 @@ lib_drv_hydra = static_library( lib_drv_ns = static_library( 'drv_ns', files( - 'north_star/distortion/utility_northstar.h', - 'north_star/distortion/deformation_northstar.h', - 'north_star/distortion/deformation_northstar.cpp', + 'north_star/distortion_3d/utility_northstar.h', + 'north_star/distortion_3d/deformation_northstar.h', + 'north_star/distortion_3d/deformation_northstar.cpp', 'north_star/ns_hmd.h', 'north_star/ns_hmd.c', 'north_star/ns_interface.h', @@ -58,16 +73,31 @@ lib_drv_ns = static_library( build_by_default: 'ns' in drivers, ) +lib_drv_ulv2 = static_library( + 'drv_ulv2', + files( + 'ultraleap_v2/ulv2_driver.cpp', + 'ultraleap_v2/ulv2_interface.h', + ), + include_directories: [xrt_include, inc_leap], + dependencies: [aux, leap], + build_by_default: 'ulv2' in drivers, +) + lib_drv_ht = static_library( 'drv_ht', files( - 'ht/ht_driver.c', - 'ht/ht_driver.h', + 'ht/ht_driver.cpp', + 'ht/ht_driver.hpp', 'ht/ht_interface.h', - 'ht/ht_prober.c', + 'ht/ht_models.hpp', + 'ht/ht_hand_math.hpp', + 'ht/ht_image_math.hpp', + 'ht/ht_nms.hpp', + 'ht/templates/NaivePermutationSort.hpp', ), - include_directories: xrt_include, - dependencies: [aux], + include_directories: [xrt_include, cjson_include], + dependencies: [aux, opencv, onnxruntime, eigen3], build_by_default: 'handtracking' in drivers, ) @@ -112,10 +142,13 @@ lib_drv_psvr = static_library( lib_drv_rs = static_library( 'drv_rs', files( - 'realsense/rs_6dof.c', + 'realsense/rs_ddev.c', + 'realsense/rs_hdev.c', 'realsense/rs_interface.h', + 'realsense/rs_driver.h', + 'realsense/rs_prober.c', ), - include_directories: xrt_include, + include_directories: [xrt_include,cjson_include], dependencies: [aux, rs], build_by_default: 'rs' in drivers, ) @@ -144,6 +177,11 @@ lib_drv_vf = static_library( build_by_default: 'vf' in drivers, ) +drv_vf = declare_dependency( + include_directories: drv_include, + link_with: lib_drv_vf, +) + lib_drv_v4l2 = static_library( 'drv_v4l2', files( @@ -165,16 +203,15 @@ lib_drv_vive = static_library( 'vive/vive_prober.c', 'vive/vive_controller.c', 'vive/vive_controller.h', - 'vive/vive_config.c', - 'vive/vive_config.h', 'vive/vive_lighthouse.c', 'vive/vive_lighthouse.h', ), include_directories: [ xrt_include, + aux_include, cjson_include, ], - dependencies: [aux, zlib], + dependencies: [aux, zlib, aux_vive], build_by_default: 'vive' in drivers, ) @@ -182,15 +219,16 @@ lib_drv_survive = static_library( 'drv_survive', files( 'survive/survive_driver.c', + 'survive/survive_driver.h', 'survive/survive_interface.h', - 'survive/survive_wrap.c', - 'survive/survive_wrap.h' + 'survive/survive_prober.c', ), include_directories: [ xrt_include, + aux_include, cjson_include, ], - dependencies: [aux, zlib, survive], + dependencies: [aux, zlib, survive, aux_vive], build_by_default: 'survive' in drivers, ) @@ -222,3 +260,65 @@ lib_drv_arduino = static_library( dependencies: [dbus, aux], build_by_default: 'arduino' in drivers, ) + +lib_drv_multi = static_library( + 'drv_multi', + files( + 'multi_wrapper/multi.c', + 'multi_wrapper/multi.h' + ), + include_directories: [ + xrt_include, + ], + dependencies: [aux], + build_by_default: true, +) + +lib_drv_qwerty = static_library( + 'drv_qwerty', + files( + 'qwerty/qwerty_device.c', + 'qwerty/qwerty_device.h', + 'qwerty/qwerty_interface.h', + 'qwerty/qwerty_prober.c', + 'qwerty/qwerty_sdl.c', + ), + include_directories: xrt_include, + dependencies: [aux, sdl2], + build_by_default: 'qwerty' in drivers, +) + +drv_qwerty_include = include_directories('qwerty') + +lib_drv_wmr = static_library( + 'drv_wmr', + files( + 'wmr/wmr_common.h', + 'wmr/wmr_config.c', + 'wmr/wmr_hmd.c', + 'wmr/wmr_hmd.h', + 'wmr/wmr_interface.h', + 'wmr/wmr_prober.c', + 'wmr/wmr_protocol.c', + 'wmr/wmr_protocol.h', + ), + include_directories: [ + xrt_include, + cjson_include, + ], + dependencies: [aux], + build_by_default: 'wmr' in drivers, +) + +lib_drv_euroc = static_library( + 'drv_euroc', + files( + 'euroc/euroc_player.cpp', + 'euroc/euroc_driver.h', + 'euroc/euroc_device.c', + 'euroc/euroc_interface.h', + ), + include_directories: [xrt_include], + dependencies: [aux, opencv], + build_by_default: 'euroc' in drivers, +) diff --git a/src/xrt/drivers/multi_wrapper/multi.c b/src/xrt/drivers/multi_wrapper/multi.c new file mode 100644 index 000000000..83498a1e9 --- /dev/null +++ b/src/xrt/drivers/multi_wrapper/multi.c @@ -0,0 +1,259 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: Apache-2.0 +/*! + * @file + * @brief Combination of multiple @ref xrt_device. + * @author Christoph Haag + * @ingroup drv_multi + */ + +#include "multi.h" +#include "util/u_device.h" +#include "util/u_debug.h" + +#include "math/m_api.h" +#include "math/m_space.h" + +DEBUG_GET_ONCE_LOG_OPTION(multi_log, "MULTI_LOG", U_LOGGING_WARN) + +#define MULTI_TRACE(d, ...) U_LOG_XDEV_IFL_T(&d->base, d->ll, __VA_ARGS__) +#define MULTI_DEBUG(d, ...) U_LOG_XDEV_IFL_D(&d->base, d->ll, __VA_ARGS__) +#define MULTI_INFO(d, ...) U_LOG_XDEV_IFL_I(&d->base, d->ll, __VA_ARGS__) +#define MULTI_WARN(d, ...) U_LOG_XDEV_IFL_W(&d->base, d->ll, __VA_ARGS__) +#define MULTI_ERROR(d, ...) U_LOG_XDEV_IFL_E(&d->base, d->ll, __VA_ARGS__) + +struct multi_device +{ + struct xrt_device base; + enum u_logging_level ll; + + struct + { + struct xrt_device *target; + struct xrt_device *tracker; + enum xrt_input_name input_name; + struct xrt_pose offset_inv; + } tracking_override; + + enum xrt_tracking_override_type override_type; +}; + +static void +direct_override(struct multi_device *d, + struct xrt_space_relation *tracker_relation, + struct xrt_space_relation *out_relation) +{ + struct xrt_space_graph xsg = {0}; + m_space_graph_add_pose_if_not_identity(&xsg, &d->tracking_override.offset_inv); + m_space_graph_add_relation(&xsg, tracker_relation); + m_space_graph_resolve(&xsg, out_relation); +} + +static void +attached_override(struct multi_device *d, + struct xrt_space_relation *target_relation, + struct xrt_pose *target_offset, + struct xrt_space_relation *tracker_relation, + struct xrt_pose *tracker_offset, + struct xrt_space_relation *in_target_space, + struct xrt_space_relation *out_relation) +{ + /* Example: + * - target: hand tracking xrt_device + * - tracker: positional tracker that the target is physically attached to + * - in_target_space: a tracked hand, relative to target's tracking origin + */ + + // XXX TODO tracking origin offsets + // m_space_graph_add_inverted_pose_if_not_identity(&xsg, tracker_offset); + // m_space_graph_add_inverted_relation(&xsg, tracker_relation); + + struct xrt_space_graph xsg = {0}; + m_space_graph_add_relation(&xsg, target_relation); + m_space_graph_add_pose_if_not_identity(&xsg, &d->tracking_override.offset_inv); + m_space_graph_add_relation(&xsg, tracker_relation); + m_space_graph_add_pose_if_not_identity(&xsg, tracker_offset); + m_space_graph_add_relation(&xsg, in_target_space); + m_space_graph_resolve(&xsg, out_relation); +} + +static void +get_tracked_pose(struct xrt_device *xdev, + enum xrt_input_name name, + uint64_t at_timestamp_ns, + struct xrt_space_relation *out_relation) +{ + struct multi_device *d = (struct multi_device *)xdev; + struct xrt_device *tracker = d->tracking_override.tracker; + enum xrt_input_name tracker_input_name = d->tracking_override.input_name; + + struct xrt_space_relation tracker_relation; + + xrt_device_get_tracked_pose(tracker, tracker_input_name, at_timestamp_ns, &tracker_relation); + + switch (d->override_type) { + case XRT_TRACKING_OVERRIDE_DIRECT: { + direct_override(d, &tracker_relation, out_relation); + } break; + case XRT_TRACKING_OVERRIDE_ATTACHED: { + struct xrt_device *target = d->tracking_override.target; + + struct xrt_space_relation target_relation; + xrt_device_get_tracked_pose(target, name, at_timestamp_ns, &target_relation); + + + // just use the origin of the tracker space as reference frame + struct xrt_space_relation in_target_space; + m_space_relation_ident(&in_target_space); + in_target_space.relation_flags = tracker_relation.relation_flags; + + struct xrt_pose *target_offset = &d->tracking_override.target->tracking_origin->offset; + struct xrt_pose *tracker_offset = &d->tracking_override.tracker->tracking_origin->offset; + + attached_override(d, &target_relation, target_offset, &tracker_relation, tracker_offset, + &in_target_space, out_relation); + } break; + } +} + +static void +destroy(struct xrt_device *xdev) +{ + struct multi_device *d = (struct multi_device *)xdev; + + xrt_device_destroy(&d->tracking_override.target); + + // we replaced the target device with us, but no the tracker + // xrt_device_destroy(&d->tracking_override.tracker); + + free(d); +} + +static void +get_hand_tracking(struct xrt_device *xdev, + enum xrt_input_name name, + uint64_t at_timestamp_ns, + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp_ns) +{ + struct multi_device *d = (struct multi_device *)xdev; + struct xrt_device *target = d->tracking_override.target; + uint64_t real_timestamp; + xrt_device_get_hand_tracking(target, name, at_timestamp_ns, out_value, &real_timestamp); + if (!out_value->is_active) { + return; + } + + struct xrt_device *tracker = d->tracking_override.tracker; + struct xrt_space_relation tracker_relation; + xrt_device_get_tracked_pose(tracker, d->tracking_override.input_name, real_timestamp, &tracker_relation); + + + switch (d->override_type) { + case XRT_TRACKING_OVERRIDE_DIRECT: { + // XXX: Codepath not tested. Probably doesn't do what you want. + direct_override(d, &out_value->hand_pose, &out_value->hand_pose); + + } break; + case XRT_TRACKING_OVERRIDE_ATTACHED: { + + // struct xrt_space_relation target_relation; + // xrt_device_get_tracked_pose(target, name, at_timestamp_ns, &target_relation); + + + // just use the origin of the tracker space as reference frame + struct xrt_space_relation in_target_space; + m_space_relation_ident(&in_target_space); + in_target_space.relation_flags = tracker_relation.relation_flags; + + struct xrt_pose *target_offset = &d->tracking_override.target->tracking_origin->offset; + struct xrt_pose *tracker_offset = &d->tracking_override.tracker->tracking_origin->offset; + + attached_override(d, &out_value->hand_pose, target_offset, &tracker_relation, tracker_offset, + &in_target_space, &out_value->hand_pose); + } break; + } +} + +static void +set_output(struct xrt_device *xdev, enum xrt_output_name name, union xrt_output_value *value) +{ + struct multi_device *d = (struct multi_device *)xdev; + struct xrt_device *target = d->tracking_override.target; + xrt_device_set_output(target, name, value); +} + +static void +get_view_pose(struct xrt_device *xdev, + const struct xrt_vec3 *eye_relation, + uint32_t view_index, + struct xrt_pose *out_pose) +{ + struct multi_device *d = (struct multi_device *)xdev; + struct xrt_device *target = d->tracking_override.target; + xrt_device_get_view_pose(target, eye_relation, view_index, out_pose); +} + +static bool +compute_distortion(struct xrt_device *xdev, int view, float u, float v, struct xrt_uv_triplet *result) +{ + struct multi_device *d = (struct multi_device *)xdev; + struct xrt_device *target = d->tracking_override.target; + return target->compute_distortion(target, view, u, v, result); +} + +static void +update_inputs(struct xrt_device *xdev) +{ + struct multi_device *d = (struct multi_device *)xdev; + struct xrt_device *target = d->tracking_override.target; + xrt_device_update_inputs(target); +} + + +struct xrt_device * +multi_create_tracking_override(enum xrt_tracking_override_type override_type, + struct xrt_device *tracking_override_target, + struct xrt_device *tracking_override_tracker, + enum xrt_input_name tracking_override_input_name, + struct xrt_pose *offset) +{ + struct multi_device *d = U_TYPED_CALLOC(struct multi_device); + + if (!d) { + return NULL; + } + + d->ll = debug_get_log_option_multi_log(); + d->override_type = override_type; + + // mimic the tracking override target + d->base = *tracking_override_target; + + // but take orientation and position tracking capabilities from tracker + d->base.orientation_tracking_supported = tracking_override_tracker->orientation_tracking_supported; + d->base.position_tracking_supported = tracking_override_tracker->position_tracking_supported; + + // because we use the tracking data of the tracker, we use its tracking origin instead + d->base.tracking_origin = tracking_override_tracker->tracking_origin; + + // The offset describes the physical pose of the tracker in the space of the thing we want to track. + // For a tracker that is physically attached at y=.1m to the tracked thing, when querying the pose for the + // tracked thing, we want to transform its pose by y-=.1m relative to the tracker. Multiple target devices may + // share a single tracker, therefore we can not simply adjust the tracker's tracking origin. + math_pose_invert(offset, &d->tracking_override.offset_inv); + + d->tracking_override.target = tracking_override_target; + d->tracking_override.tracker = tracking_override_tracker; + d->tracking_override.input_name = tracking_override_input_name; + + d->base.get_tracked_pose = get_tracked_pose; + d->base.destroy = destroy; + d->base.get_hand_tracking = get_hand_tracking; + d->base.set_output = set_output; + d->base.update_inputs = update_inputs; + d->base.compute_distortion = compute_distortion; + d->base.get_view_pose = get_view_pose; + + return &d->base; +} diff --git a/src/xrt/drivers/multi_wrapper/multi.h b/src/xrt/drivers/multi_wrapper/multi.h new file mode 100644 index 000000000..3be7244e1 --- /dev/null +++ b/src/xrt/drivers/multi_wrapper/multi.h @@ -0,0 +1,53 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: Apache-2.0 +/*! + * @file + * @brief Combination of multiple @ref xrt_device. + * @author Christoph Haag + * @ingroup drv_multi + */ + +#pragma once + +#include "xrt/xrt_defines.h" +#include "xrt/xrt_device.h" +#include "xrt/xrt_settings.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +/*! + * @defgroup drv_multi Multi device wrapper driver + * @ingroup drv + * + * @brief Driver that can wrap multiple devices, for example to override tracking. + */ + +/*! + * Create a device that takes ownership of the target device and mimics it. + * + * Does not take ownership of the tracker device, one can be assigned to multiple targets. + * + * The pose provided by get_tracked_pose will be provided by the tracker device. + * + * @param tracking_override_target An existing device that will be mimiced by the created device. + * @param tracking_override_tracker An existing device that will be used to provide tracking data. + * @param tracking_override_input_name The input name of the tracker device. XRT_INPUT_GENERIC_TRACKER_POSE for generic + * trackers. + * @param offset A static offset describing the real world transform from the "tracked point" of the target device to + * the "tracked point" of the tracker device. A tracking sensors attached .1m above the HMD "center" sets y = 0.1. + * + * @ingroup drv_multi + */ +struct xrt_device * +multi_create_tracking_override(enum xrt_tracking_override_type override_type, + struct xrt_device *tracking_override_target, + struct xrt_device *tracking_override_tracker, + enum xrt_input_name tracking_override_input_name, + struct xrt_pose *offset); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/north_star/distortion/deformation_northstar.cpp b/src/xrt/drivers/north_star/distortion_3d/deformation_northstar.cpp similarity index 97% rename from src/xrt/drivers/north_star/distortion/deformation_northstar.cpp rename to src/xrt/drivers/north_star/distortion_3d/deformation_northstar.cpp index e816eedf9..a4c48bdae 100644 --- a/src/xrt/drivers/north_star/distortion/deformation_northstar.cpp +++ b/src/xrt/drivers/north_star/distortion_3d/deformation_northstar.cpp @@ -20,7 +20,7 @@ OpticalSystem::OpticalSystem(const OpticalSystem &_in) } void -OpticalSystem::LoadOpticalData(struct ns_v1_eye *eye) +OpticalSystem::LoadOpticalData(struct ns_3d_eye *eye) { ellipseMinorAxis = eye->ellipse_minor_axis; @@ -257,9 +257,8 @@ OpticalSystem::DisplayUVToRenderUVPreviousSeed(Vector2 inputUV) return curDisplayUV; } - extern "C" struct ns_optical_system * -ns_create_optical_system(struct ns_v1_eye *eye) +ns_3d_create_optical_system(struct ns_3d_eye *eye) { OpticalSystem *opticalSystem = new OpticalSystem(); opticalSystem->LoadOpticalData(eye); @@ -269,11 +268,11 @@ ns_create_optical_system(struct ns_v1_eye *eye) } extern "C" void -ns_display_uv_to_render_uv(struct ns_uv in, struct ns_uv *out, struct ns_v1_eye *eye) +ns_3d_display_uv_to_render_uv(struct xrt_vec2 in, struct xrt_vec2 *out, struct ns_3d_eye *eye) { OpticalSystem *opticalSystem = (OpticalSystem *)eye->optical_system; - Vector2 inUV = Vector2(in.u, 1.f - in.v); + Vector2 inUV = Vector2(in.x, 1.f - in.y); Vector2 outUV = opticalSystem->DisplayUVToRenderUVPreviousSeed(inUV); - out->u = outUV.x; - out->v = outUV.y; + out->x = outUV.x; + out->y = outUV.y; } diff --git a/src/xrt/drivers/north_star/distortion/deformation_northstar.h b/src/xrt/drivers/north_star/distortion_3d/deformation_northstar.h similarity index 98% rename from src/xrt/drivers/north_star/distortion/deformation_northstar.h rename to src/xrt/drivers/north_star/distortion_3d/deformation_northstar.h index dd646b3fd..5b24bb44f 100644 --- a/src/xrt/drivers/north_star/distortion/deformation_northstar.h +++ b/src/xrt/drivers/north_star/distortion_3d/deformation_northstar.h @@ -17,7 +17,7 @@ public: OpticalSystem(const OpticalSystem &_in); void - LoadOpticalData(struct ns_v1_eye *eye); + LoadOpticalData(struct ns_3d_eye *eye); Vector3 GetEyePosition() diff --git a/src/xrt/drivers/north_star/distortion/utility_northstar.h b/src/xrt/drivers/north_star/distortion_3d/utility_northstar.h similarity index 94% rename from src/xrt/drivers/north_star/distortion/utility_northstar.h rename to src/xrt/drivers/north_star/distortion_3d/utility_northstar.h index c7c2e5848..af73701e8 100644 --- a/src/xrt/drivers/north_star/distortion/utility_northstar.h +++ b/src/xrt/drivers/north_star/distortion_3d/utility_northstar.h @@ -8,7 +8,6 @@ using namespace std; // min set of functions needed -const float LP_PI = 3.14159265f; // float error, will be 3.141592741f. const float kEpsilon = 0.00001f; @@ -857,7 +856,7 @@ public: float sinp = +2.0f * (q.w * q.y - q.z * q.x); if (fabs(sinp) >= 1.f) { - pitch = copysignf(LP_PI / 2.f, sinp); + pitch = copysignf(M_PI / 2.f, sinp); } else { pitch = asinf(sinp); } @@ -871,7 +870,7 @@ public: ToEulerAngles(const Quaternion &in) { Vector3 euler; - const static float PI_OVER_2 = LP_PI * 0.5f; + const static float PI_OVER_2 = M_PI * 0.5f; const static float EPSILON = 1e-10f; float sqw, sqx, sqy, sqz; @@ -892,7 +891,7 @@ public: // If facing down, reverse yaw if (euler.y < 0.f) { - euler.z = LP_PI - euler.z; + euler.z = M_PI - euler.z; } } return euler; @@ -1179,65 +1178,3 @@ public: float z; float w; }; - -// A position and rotation. You can multiply two poses; this acts like -// Matrix4x4 multiplication, but Poses always have unit scale. -class Pose -{ -public: - Vector3 position; - Quaternion rotation; - - explicit Pose(Vector3 const &pos) - { - position = pos; - rotation = Quaternion::Identity(); - } - explicit Pose(Quaternion const &rot) - { - position = Vector3::Zero(); - rotation = rot; - } - Pose(Vector3 const &pos, Quaternion const &rot) - { - position = pos; - rotation = rot; - } - - inline static Pose - Identity() - { - return Pose(Vector3::Zero(), Quaternion::Identity()); - } - - inline Pose - Inverse() const - { - Quaternion invQ = rotation.Inverse(); - return Pose(invQ * -position, invQ); - } - - inline Matrix4x4 - Matrix() const - { - return Matrix4x4::Translate(position) * rotation.ToMatrix4x4(); - } - - inline Pose // Until clang-format-11 is on the CI. - operator*(Pose const &rhs) const - { - return Pose(position + (rotation * rhs.position), rotation * rhs.rotation); - } - - inline Pose // Until clang-format-11 is on the CI. - operator*(Vector3 const &rhs) const - { - return Pose(position + rotation * rhs, rotation); - } - - inline static Pose - FromMatrix(Matrix4x4 m) - { - return Pose(Vector3(m.m03, m.m13, m.m23), Quaternion::FromMatrix(m)); - } -}; diff --git a/src/xrt/drivers/north_star/ns_hmd.c b/src/xrt/drivers/north_star/ns_hmd.c index a3c080367..403ccd2b3 100644 --- a/src/xrt/drivers/north_star/ns_hmd.c +++ b/src/xrt/drivers/north_star/ns_hmd.c @@ -8,6 +8,7 @@ * @author Nova King * @author Jakob Bornecrantz * @author Moses Turner + * @author Nico Zobernig * @ingroup drv_ns */ @@ -26,16 +27,395 @@ #include "util/u_debug.h" #include "util/u_device.h" #include "util/u_time.h" -#include "util/u_distortion_mesh.h" -#include "xrt/xrt_config_drivers.h" -#ifdef XRT_BUILD_DRIVER_RS -#include "../realsense/rs_interface.h" -#endif +#include "math/m_space.h" DEBUG_GET_ONCE_LOG_OPTION(ns_log, "NS_LOG", U_LOGGING_INFO) -struct xrt_pose t265_to_nose_bridge = {.orientation = {0, 0, 0, 1}, .position = {0, 0, 0}}; +#define printf_pose(pose) \ + printf("%f %f %f %f %f %f %f\n", pose.position.x, pose.position.y, pose.position.z, pose.orientation.x, \ + pose.orientation.y, pose.orientation.z, pose.orientation.w); + + +static float +try_get_ipd(struct ns_hmd *ns, const struct cJSON *json) +{ // + const char *things[] = {"baseline", "ipd", "IPD"}; + bool done = false; + float out; + const char *thing; + for (int i = 0; (!done && (i < 3)); i++) { + thing = things[i]; + done = u_json_get_float(u_json_get(json, thing), &out); + } + if (!done) { + NS_INFO(ns, + "No key `baseline (or ipd, or IPD)` in your config file. Guessing the IPD is 64 millimeters"); + out = 64.0f; + } + if (out > 250.0f) { + NS_ERROR(ns, "IPD is way too high (%f millimeters!) Are you sure `%s` in your config file is correct?", + out, thing); + } + if (out < 10.0f) { + NS_ERROR(ns, "IPD is way too low (%f millimeters!) Are you sure `%s` in your config file is correct?", + out, thing); + } + out *= 0.001f; + NS_DEBUG(ns, "IPD returned is %f meters", out); + + return out; +} + +static void +try_get_fov(struct ns_hmd *ns, const struct cJSON *json, struct xrt_fov *left_fov, struct xrt_fov *right_fov) +{ + const char *things[] = {"fov", "FOV"}; + float out_float; + struct xrt_fov out_fov; + const char *thing; + for (int i = 0; (i < 2); i++) { + thing = things[i]; + const cJSON *fov_obj = u_json_get(json, thing); + if (fov_obj == NULL) { + continue; + } + if (u_json_get_float_array(fov_obj, &out_fov.angle_left, 4)) { // LRTB array of floats, this is allowed. + goto good; + } + if (u_json_get_float(fov_obj, &out_float)) { + out_fov.angle_left = -out_float; + out_fov.angle_right = out_float; + out_fov.angle_up = out_float; + out_fov.angle_down = -out_float; + goto good; + } + } + // Defaults, get skipped over if we found a FOV in the json + NS_INFO(ns, "No key `fov` in your config file. Guessing you want 0.7 radian half-angles."); + out_fov.angle_left = -0.7f; + out_fov.angle_right = 0.7f; + out_fov.angle_up = 0.7f; + out_fov.angle_down = -0.7f; + +good: + assert(out_fov.angle_right > out_fov.angle_left); + assert(out_fov.angle_up > out_fov.angle_down); + assert(fabsf(out_fov.angle_up) < M_PI_2); + assert(fabsf(out_fov.angle_down) < M_PI_2); + assert(fabsf(out_fov.angle_left) < M_PI_2); + assert(fabsf(out_fov.angle_right) < M_PI_2); + memcpy(left_fov, &out_fov, sizeof(struct xrt_fov)); + memcpy(right_fov, &out_fov, sizeof(struct xrt_fov)); + return; +} + +bool +ns_vipd_mesh_calc(struct xrt_device *xdev, int view, float u, float v, struct xrt_uv_triplet *result) +{ + struct ns_hmd *ns = ns_hmd(xdev); + return u_compute_distortion_ns_vipd(&ns->dist_vipd, view, u, v, result); +} + +bool +ns_vipd_parse(struct ns_hmd *ns) +{ + + struct u_ns_vipd_values *temp_data = &ns->dist_vipd; + const struct cJSON *config_json = ns->config_json; + + const cJSON *grids_json = u_json_get(config_json, "grids"); + if (grids_json == NULL) + goto cleanup_vipd; + + const cJSON *current_element = NULL; + char *current_key = NULL; + + cJSON_ArrayForEach(current_element, grids_json) + { // Note to people reviewing this: this is definitely not super safe. Tried to add as many null-checks as + // possible etc. but is probably a waste of time, it takes a while to do this right and the only person using + // this code is me -Moses + current_key = current_element->string; + float ipd = strtof(current_key, NULL) / 1000; + if (!((ipd < .100) && (ipd > .030))) { + U_LOG_E("Nonsense IPD in grid %d, skipping", temp_data->number_of_ipds + 1); + continue; + } + + temp_data->number_of_ipds += 1; + temp_data->ipds = realloc(temp_data->ipds, temp_data->number_of_ipds * sizeof(float)); + temp_data->ipds[temp_data->number_of_ipds - 1] = ipd; + temp_data->grids = realloc(temp_data->grids, temp_data->number_of_ipds * sizeof(struct u_ns_vipd_grid)); + + for (int view = 0; view <= 1; view++) { + const struct cJSON *grid_root = u_json_get(current_element, view ? "right" : "left"); + // if view is 0, then left. if view is 1, then right + for (int lv = 0; lv < 65; lv++) { + struct cJSON *v_axis = cJSON_GetArrayItem(grid_root, lv); + + for (int lu = 0; lu < 65; lu++) { + struct cJSON *cell = cJSON_GetArrayItem(v_axis, lu + 1); + + struct cJSON *cellX = cJSON_GetArrayItem(cell, 0); + struct cJSON *cellY = cJSON_GetArrayItem(cell, 1); + if (grid_root == NULL || cell == NULL || v_axis == NULL || cellX == NULL || + cellY == NULL) { + NS_ERROR(ns, + "VIPD distortion config is malformed in some way, bailing."); + goto cleanup_vipd; + } + temp_data->grids[temp_data->number_of_ipds - 1].grid[view][lv][lu].x = + (float)cellX->valuedouble; + temp_data->grids[temp_data->number_of_ipds - 1].grid[view][lv][lu].y = + (float)cellY->valuedouble; + } + } + } + } + + float baseline = try_get_ipd(ns, config_json); + + struct u_ns_vipd_grid *high_grid = {0}; + struct u_ns_vipd_grid *low_grid = {0}; + float interp = 0; + for (int i = 1; i < temp_data->number_of_ipds; i++) { + NS_DEBUG(ns, "looking at %f lower and %f upper\n", temp_data->ipds[i - 1], temp_data->ipds[i]); + if ((baseline >= temp_data->ipds[i - 1]) && (baseline <= temp_data->ipds[i])) { + NS_DEBUG(ns, "okay, IPD is between %f and %f\n", temp_data->ipds[i - 1], temp_data->ipds[i]); + high_grid = &temp_data->grids[i - 1]; + low_grid = &temp_data->grids[i]; + interp = math_map_ranges(baseline, temp_data->ipds[i - 1], temp_data->ipds[i], 0, 1); + NS_DEBUG(ns, "interp is %f\n", interp); + break; + } + } + + for (int view = 0; view <= 1; view++) { + for (int lv = 0; lv < 65; lv++) { + for (int lu = 0; lu < 65; lu++) { + temp_data->grid_for_use.grid[view][lv][lu].x = math_map_ranges( + interp, 0, 1, low_grid->grid[view][lv][lu].x, high_grid->grid[view][lv][lu].x); + temp_data->grid_for_use.grid[view][lv][lu].y = math_map_ranges( + interp, 0, 1, low_grid->grid[view][lv][lu].y, high_grid->grid[view][lv][lu].y); + } + } + } + + try_get_fov(ns, config_json, &temp_data->fov[0], &temp_data->fov[1]); + + // stupid + memcpy(&ns->base.hmd->views[0].fov, &temp_data->fov[0], sizeof(struct xrt_fov)); + memcpy(&ns->base.hmd->views[1].fov, &temp_data->fov[1], sizeof(struct xrt_fov)); + + printf("%f %f %f %f\n", ns->base.hmd->views[1].fov.angle_down, ns->base.hmd->views[1].fov.angle_left, + ns->base.hmd->views[1].fov.angle_right, ns->base.hmd->views[1].fov.angle_up); + + ns->head_pose_to_eye[0].orientation.x = 0.0f; + ns->head_pose_to_eye[0].orientation.y = 0.0f; + ns->head_pose_to_eye[0].orientation.z = 0.0f; + ns->head_pose_to_eye[0].orientation.w = 1.0f; + ns->head_pose_to_eye[0].position.x = -baseline / 2; + ns->head_pose_to_eye[0].position.y = 0.0f; + ns->head_pose_to_eye[0].position.z = 0.0f; + + + + ns->head_pose_to_eye[1].orientation.x = 0.0f; + ns->head_pose_to_eye[1].orientation.y = 0.0f; + ns->head_pose_to_eye[1].orientation.z = 0.0f; + ns->head_pose_to_eye[1].orientation.w = 1.0f; + ns->head_pose_to_eye[1].position.x = baseline / 2; + ns->head_pose_to_eye[1].position.y = 0.0f; + ns->head_pose_to_eye[1].position.z = 0.0f; + + ns->base.compute_distortion = &ns_vipd_mesh_calc; + + return true; + +cleanup_vipd: + memset(&ns->dist_vipd, 0, sizeof(struct u_ns_vipd_values)); + return false; +} + +/* + * + * "2D Polynomial" distortion; original implementation by Johnathon Zelstadt + * Sometimes known as "v2", filename is often NorthStarCalibration.json + * + */ + +static bool +ns_p2d_mesh_calc(struct xrt_device *xdev, int view, float u, float v, struct xrt_uv_triplet *result) +{ + struct ns_hmd *ns = ns_hmd(xdev); + return u_compute_distortion_ns_p2d(&ns->dist_p2d, view, u, v, result); +} + + +bool +ns_p2d_parse(struct ns_hmd *ns) +{ + + struct xrt_pose temp_eyes_center_to_eye[2]; + + // convenience names + const struct cJSON *config_json = ns->config_json; + + // Note that x and y are flipped. We have to flip 'em at some point - the polynomial calibrator has a strange + // definition of x and y. "opencv treats column major over row major (as in, Y,X for image look up)" -Dr. Damo + if (u_json_get_float_array(u_json_get(config_json, "left_uv_to_rect_x"), ns->dist_p2d.y_coefficients_right, + 16) != 16) + goto cleanup_p2d; + if (u_json_get_float_array(u_json_get(config_json, "left_uv_to_rect_y"), ns->dist_p2d.x_coefficients_right, + 16) != 16) + goto cleanup_p2d; + if (u_json_get_float_array(u_json_get(config_json, "right_uv_to_rect_x"), ns->dist_p2d.y_coefficients_left, + 16) != 16) + goto cleanup_p2d; + if (u_json_get_float_array(u_json_get(config_json, "right_uv_to_rect_y"), ns->dist_p2d.x_coefficients_left, + 16) != 16) + goto cleanup_p2d; + + // at this point, locked into using this distortion method - we can touch anything and not worry about side + // effects + float baseline = try_get_ipd(ns, config_json); + + math_pose_identity(&temp_eyes_center_to_eye[0]); + math_pose_identity(&temp_eyes_center_to_eye[1]); + temp_eyes_center_to_eye[0].position.x = -baseline / 2; + temp_eyes_center_to_eye[1].position.x = baseline / 2; + + try_get_fov(ns, config_json, &ns->dist_p2d.fov[0], &ns->dist_p2d.fov[1]); + + memcpy(&ns->base.hmd->views[0].fov, &ns->dist_p2d.fov[0], sizeof(struct xrt_fov)); + memcpy(&ns->base.hmd->views[1].fov, &ns->dist_p2d.fov[1], sizeof(struct xrt_fov)); + + ns->base.compute_distortion = &ns_p2d_mesh_calc; + memcpy(&ns->head_pose_to_eye, &temp_eyes_center_to_eye, sizeof(struct xrt_pose) * 2); + + return true; + +cleanup_p2d: + memset(&ns->dist_p2d, 0, sizeof(struct u_ns_p2d_values)); + return false; +} + + +/* + * + * "Original 3D" undistortion, by Leap Motion + * Sometimes known as "v1", config file name is often "Calibration.json" + * + */ + +static bool +ns_3d_mesh_calc(struct xrt_device *xdev, int view, float u, float v, struct xrt_uv_triplet *result) +{ + struct ns_hmd *ns = ns_hmd(xdev); + struct ns_3d_data *data = &ns->dist_3d; + struct xrt_vec2 uv = {u, v}; + struct xrt_vec2 warped_uv = {0.0f, 0.0f}; + + ns_3d_display_uv_to_render_uv(uv, &warped_uv, &data->eyes[view]); + + result->r.x = warped_uv.x; + result->r.y = warped_uv.y; + result->g.x = warped_uv.x; + result->g.y = warped_uv.y; + result->b.x = warped_uv.x; + result->b.y = warped_uv.y; + return true; +} + +static void +ns_3d_fov_calculate(struct xrt_fov *fov, struct xrt_quat projection) +{ + // Million thanks to Nico Zobernig for figuring this out + fov->angle_left = atanf(projection.x); + fov->angle_right = atanf(projection.y); + fov->angle_up = atanf(projection.z); + fov->angle_down = atanf(projection.w); +} + +/* + * + * Parse functions. + * + */ + +static bool +ns_3d_leap_parse(struct ns_3d_leap *leap, const struct cJSON *leap_data) +{ + u_json_get_string_into_array(u_json_get(leap_data, "name"), leap->name, 64); + u_json_get_string_into_array(u_json_get(leap_data, "serial"), leap->serial, 64); + if (!u_json_get_vec3(u_json_get(u_json_get(leap_data, "localPose"), "position"), &leap->pose.position)) + return false; + if (!u_json_get_quat(u_json_get(u_json_get(leap_data, "localPose"), "rotation"), &leap->pose.orientation)) + return false; + return true; +} + +static bool +ns_3d_eye_parse(struct ns_3d_eye *eye, const struct cJSON *eye_data) +{ + if (!u_json_get_float(u_json_get(eye_data, "ellipseMinorAxis"), &eye->ellipse_minor_axis)) + return false; + if (!u_json_get_float(u_json_get(eye_data, "ellipseMajorAxis"), &eye->ellipse_major_axis)) + return false; + if (!u_json_get_vec3(u_json_get(eye_data, "screenForward"), &eye->screen_forward)) + return false; + if (!u_json_get_vec3(u_json_get(eye_data, "screenPosition"), &eye->screen_position)) + return false; + if (!u_json_get_vec3(u_json_get(eye_data, "eyePosition"), &eye->eye_pose.position)) + return false; + if (!u_json_get_quat(u_json_get(eye_data, "eyeRotation"), &eye->eye_pose.orientation)) + return false; + if (!u_json_get_quat(u_json_get(eye_data, "cameraProjection"), &eye->camera_projection)) + return false; + for (int x = 0; x < 4; ++x) { + for (int y = 0; y < 4; ++y) { + char key[4]; + sprintf(key, "e%d%d", x, y); + + u_json_get_float(u_json_get(u_json_get(eye_data, "sphereToWorldSpace"), key), + &eye->sphere_to_world_space.v[(x * 4) + y]); + u_json_get_float(u_json_get(u_json_get(eye_data, "worldToScreenSpace"), key), + &eye->world_to_screen_space.v[(x * 4) + y]); + } + } + return true; +} + +bool +ns_3d_parse(struct ns_hmd *ns) +{ + struct ns_3d_data *our_ns_3d_data = &ns->dist_3d; + + if (!ns_3d_eye_parse(&our_ns_3d_data->eyes[0], u_json_get(ns->config_json, "leftEye"))) + goto cleanup_l3d; + if (!ns_3d_eye_parse(&our_ns_3d_data->eyes[1], u_json_get(ns->config_json, "rightEye"))) + goto cleanup_l3d; + if (!ns_3d_leap_parse(&our_ns_3d_data->leap, u_json_get(ns->config_json, "leapTracker"))) + goto cleanup_l3d; + + // Locked in, okay to touch anything inside ns struct + ns_3d_fov_calculate(&ns->base.hmd->views[0].fov, our_ns_3d_data->eyes[0].camera_projection); + ns_3d_fov_calculate(&ns->base.hmd->views[1].fov, our_ns_3d_data->eyes[1].camera_projection); + + ns->head_pose_to_eye[0] = our_ns_3d_data->eyes[0].eye_pose; // Left eye. + ns->head_pose_to_eye[1] = our_ns_3d_data->eyes[1].eye_pose; // Right eye. + + our_ns_3d_data->eyes[0].optical_system = ns_3d_create_optical_system(&our_ns_3d_data->eyes[0]); + our_ns_3d_data->eyes[1].optical_system = ns_3d_create_optical_system(&our_ns_3d_data->eyes[1]); + + ns->base.compute_distortion = &ns_3d_mesh_calc; + + return true; + +cleanup_l3d: + memset(&ns->dist_3d, 0, sizeof(struct ns_3d_data)); + return false; +} /* * @@ -43,14 +423,6 @@ struct xrt_pose t265_to_nose_bridge = {.orientation = {0, 0, 0, 1}, .position = * */ -static double -map(double value, double fromLow, double fromHigh, double toLow, double toHigh) -{ - return (value - fromLow) * (toHigh - toLow) / (fromHigh - fromLow) + toLow; -} // Math copied from - // https://www.arduino.cc/reference/en/language/functions/math/map/ -// This is pure math so it is still under the Boost Software License. - static void ns_hmd_destroy(struct xrt_device *xdev) { @@ -64,19 +436,7 @@ ns_hmd_destroy(struct xrt_device *xdev) static void ns_hmd_update_inputs(struct xrt_device *xdev) -{ - struct ns_hmd *ns = ns_hmd(xdev); - - if (ns->tracker != NULL) { - xrt_device_update_inputs(ns->tracker); - } -} - -/* - * - * V1 functions - * - */ +{} static void ns_hmd_get_tracked_pose(struct xrt_device *xdev, @@ -86,393 +446,85 @@ ns_hmd_get_tracked_pose(struct xrt_device *xdev, { struct ns_hmd *ns = ns_hmd(xdev); - - // If the tracking device is created use it. - if (ns->tracker != NULL) { - xrt_device_get_tracked_pose(ns->tracker, name, at_timestamp_ns, out_relation); - return; - } - if (name != XRT_INPUT_GENERIC_HEAD_POSE) { NS_ERROR(ns, "unknown input name"); return; } - out_relation->pose = ns->pose; - out_relation->relation_flags = (enum xrt_space_relation_flags)(XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | - XRT_SPACE_RELATION_POSITION_VALID_BIT | - XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT); + *out_relation = ns->no_tracker_relation; // you can change this using the debug gui } static void ns_hmd_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { struct ns_hmd *ns = ns_hmd(xdev); - *out_pose = ns->eye_configs_v1[view_index].eye_pose; + *out_pose = ns->head_pose_to_eye[view_index]; } - -/* - * - * V1 Mesh functions. - * - */ - static bool -ns_mesh_calc(struct xrt_device *xdev, int view, float u, float v, struct xrt_uv_triplet *result) -{ - struct ns_hmd *ns = ns_hmd(xdev); - - struct ns_uv uv = {u, v}; - struct ns_uv warped_uv = {0.0f, 0.0f}; - ns_display_uv_to_render_uv(uv, &warped_uv, &ns->eye_configs_v1[view]); - - result->r.x = warped_uv.u; - result->r.y = warped_uv.v; - result->g.x = warped_uv.u; - result->g.y = warped_uv.v; - result->b.x = warped_uv.u; - result->b.y = warped_uv.v; - return true; -} - -/* - * - * Parse function. - * - */ - -static void -ns_leap_parse(struct ns_leap *leap, struct cJSON *leap_data) -{ - /* - These are very wrong! - You could very likely write into random memory here. - - u_json_get_string(cJSON_GetObjectItemCaseSensitive(leap_data, - "name"), &leap->name); - u_json_get_string(cJSON_GetObjectItemCaseSensitive(leap_data, - "serial"), &leap->serial); - */ - - u_json_get_vec3( - cJSON_GetObjectItemCaseSensitive(cJSON_GetObjectItemCaseSensitive(leap_data, "localPose"), "position"), - &leap->pose.position); - u_json_get_quat( - cJSON_GetObjectItemCaseSensitive(cJSON_GetObjectItemCaseSensitive(leap_data, "localPose"), "rotation"), - &leap->pose.orientation); -} - -static void -ns_eye_parse(struct ns_v1_eye *eye, struct cJSON *eye_data) -{ - u_json_get_float(cJSON_GetObjectItemCaseSensitive(eye_data, "ellipseMinorAxis"), &eye->ellipse_minor_axis); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(eye_data, "ellipseMajorAxis"), &eye->ellipse_major_axis); - u_json_get_vec3(cJSON_GetObjectItemCaseSensitive(eye_data, "screenForward"), &eye->screen_forward); - u_json_get_vec3(cJSON_GetObjectItemCaseSensitive(eye_data, "screenPosition"), &eye->screen_position); - u_json_get_vec3(cJSON_GetObjectItemCaseSensitive(eye_data, "eyePosition"), &eye->eye_pose.position); - u_json_get_quat(cJSON_GetObjectItemCaseSensitive(eye_data, "eyeRotation"), &eye->eye_pose.orientation); - u_json_get_quat(cJSON_GetObjectItemCaseSensitive(eye_data, "cameraProjection"), &eye->camera_projection); - for (int x = 0; x < 4; ++x) { - for (int y = 0; y < 4; ++y) { - char key[4]; - sprintf(key, "e%d%d", x, y); - - u_json_get_float(cJSON_GetObjectItemCaseSensitive( - cJSON_GetObjectItemCaseSensitive(eye_data, "sphereToWorldSpace"), key), - &eye->sphere_to_world_space.v[(x * 4) + y]); - u_json_get_float(cJSON_GetObjectItemCaseSensitive( - cJSON_GetObjectItemCaseSensitive(eye_data, "worldToScreenSpace"), key), - &eye->world_to_screen_space.v[(x * 4) + y]); - } - } -} - - -static void -ns_fov_calculate(struct xrt_fov *fov, struct xrt_quat projection) -{ // All of these are wrong - gets "hidden" by the simple_fov making it look - // okay. Needs to be fixed. - - fov->angle_up = projection.x; // atanf(fabsf(projection.x) / - // near_plane); - fov->angle_down = projection.y; // atanf(fabsf(projection.y) / near_plane); - fov->angle_left = projection.z; // atanf(fabsf(projection.z) / near_plane); - fov->angle_right = projection.w; // atanf(fabsf(projection.w) / near_plane); -} - -/* - * - * V2 optics. - * - * - */ - - -static void -ns_v2_hmd_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, - uint32_t view_index, - struct xrt_pose *out_pose) -{ - // Copied from dummy driver - - struct ns_hmd *ns = ns_hmd(xdev); - - struct xrt_pose pose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; - bool adjust = view_index == 0; - - pose.position.x = ns->ipd / 2.0f; - pose.position.y = eye_relation->y / 2.0f; - pose.position.z = eye_relation->z / 2.0f; - - // Adjust for left/right while also making sure there aren't any -0.f. - if (pose.position.x > 0.0f && adjust) { - pose.position.x = -pose.position.x; - } - if (pose.position.y > 0.0f && adjust) { - pose.position.y = -pose.position.y; - } - if (pose.position.z > 0.0f && adjust) { - pose.position.z = -pose.position.z; - } - - *out_pose = pose; -} - -static float -ns_v2_polyval2d(float X, float Y, float C[16]) -{ - float X2 = X * X; - float X3 = X2 * X; - float Y2 = Y * Y; - float Y3 = Y2 * Y; - return (((C[0]) + (C[1] * Y) + (C[2] * Y2) + (C[3] * Y3)) + - ((C[4] * X) + (C[5] * X * Y) + (C[6] * X * Y2) + (C[7] * X * Y3)) + - ((C[8] * X2) + (C[9] * X2 * Y) + (C[10] * X2 * Y2) + (C[11] * X2 * Y3)) + - ((C[12] * X3) + (C[13] * X3 * Y) + (C[14] * X3 * Y2) + (C[15] * X3 * Y3))); -} - - - -static void -ns_v2_fov_calculate(struct ns_hmd *ns, int eye_index) -{ - ns->base.hmd->views[eye_index].fov.angle_down = ns->eye_configs_v2[eye_index].fov.angle_down; - ns->base.hmd->views[eye_index].fov.angle_up = ns->eye_configs_v2[eye_index].fov.angle_up; - ns->base.hmd->views[eye_index].fov.angle_left = ns->eye_configs_v2[eye_index].fov.angle_left; - ns->base.hmd->views[eye_index].fov.angle_right = ns->eye_configs_v2[eye_index].fov.angle_right; -} - - - -static bool -ns_v2_mesh_calc(struct xrt_device *xdev, int view, float u, float v, struct xrt_uv_triplet *result) -{ - /*! @todo (Moses Turner) It should not be necessary to reverse the U and - * V coords. I have no idea why it is like this. It shouldn't be like - * this. It must be something wrong with the undistortion calibrator. - * The V2 undistortion calibrator software is here if you want to look: - * https://github.com/BryanChrisBrown/ProjectNorthStar/tree/feat-gen-2-software/Software/North%20Star%20Gen%202/North%20Star%20Calibrator - */ - // u = 1.0 - u; - v = 1.0 - v; - - - struct ns_hmd *ns = ns_hmd(xdev); - - float x_ray = ns_v2_polyval2d(u, v, ns->eye_configs_v2[view].x_coefficients); - float y_ray = ns_v2_polyval2d(u, v, ns->eye_configs_v2[view].y_coefficients); - - - float left_ray_bound = tan(ns->eye_configs_v2[view].fov.angle_left); - float right_ray_bound = tan(ns->eye_configs_v2[view].fov.angle_right); - float up_ray_bound = tan(ns->eye_configs_v2[view].fov.angle_up); - float down_ray_bound = tan(ns->eye_configs_v2[view].fov.angle_down); - - float u_eye = map(x_ray, left_ray_bound, right_ray_bound, 0, 1); - - float v_eye = map(y_ray, down_ray_bound, up_ray_bound, 0, 1); - - - // boilerplate, put the UV coordinates in all the RGB slots - result->r.x = u_eye; - result->r.y = v_eye; - result->g.x = u_eye; - result->g.y = v_eye; - result->b.x = u_eye; - result->b.y = v_eye; - - return true; -} - - -static bool -ns_config_load(struct ns_hmd *ns) +ns_config_load(struct ns_hmd *ns, const char *config_path) { // Get the path to the JSON file - if (ns->config_path == NULL || strcmp(ns->config_path, "/") == 0) { - NS_ERROR(ns, - "Configuration path \"%s\" does not lead to a " - "configuration JSON file. Set the NS_CONFIG_PATH env " - "variable to your JSON.", - ns->config_path); + bool json_allocated = false; + if (config_path == NULL || strcmp(config_path, "/") == 0) { + NS_INFO(ns, + "Configuration path \"%s\" does not lead to a " + "configuration JSON file. Set the NS_CONFIG_PATH env " + "variable to your JSON.", + config_path); return false; } - // Open the JSON file and put its contents into a string - FILE *config_file = fopen(ns->config_path, "r"); + FILE *config_file = fopen(config_path, "r"); if (config_file == NULL) { - NS_ERROR(ns, "The configuration file at path \"%s\" was unable to load", ns->config_path); - return false; + NS_INFO(ns, "The configuration file at path \"%s\" was unable to load", config_path); + goto parse_error; } - char json[8192]; - size_t i = 0; - while (!feof(config_file) && i < (sizeof(json) - 1)) { - json[i++] = fgetc(config_file); + fseek(config_file, 0, SEEK_END); // Go to end of file + int file_size = ftell(config_file); // See offset we're at. This should be the file size in bytes. + rewind(config_file); // Go back to the beginning of the file + + if (file_size == 0) { + NS_INFO(ns, "Empty config file!"); + goto parse_error; + } else if (file_size > 3 * pow(1024, 2)) { // 3 MiB + NS_INFO(ns, "Huge config file! (%f MiB!!) Something's wrong here.", ((float)file_size) / pow(1024, 2)); + goto parse_error; } - json[i] = '\0'; - struct cJSON *config_json; + char *json = calloc(file_size + 1, 1); + json_allocated = true; - // Parse the JSON file - config_json = cJSON_Parse(json); - if (config_json == NULL) { + fread(json, 1, file_size, config_file); + fclose(config_file); + json[file_size] = '\0'; + + ns->config_json = cJSON_Parse(json); + if (ns->config_json == NULL) { const char *error_ptr = cJSON_GetErrorPtr(); - NS_ERROR(ns, "The JSON file at path \"%s\" was unable to parse", ns->config_path); + NS_INFO(ns, "The JSON file at path \"%s\" was unable to parse", config_path); if (error_ptr != NULL) { - NS_ERROR(ns, "because of an error before %s", error_ptr); + NS_INFO(ns, "because of an error before %s", error_ptr); } - return false; - } - if (cJSON_GetObjectItemCaseSensitive(config_json, "leftEye") == NULL && - cJSON_GetObjectItemCaseSensitive(config_json, "left_uv_to_rect_x") != NULL) { - // Bad hack to tell that we're v2. Error checking is not good - // enough for public consumption - many cases where a malformed - // config json results in cryptic errors. Should get fixed - // whenever we have more than 5 people using this. - u_json_get_float(cJSON_GetObjectItemCaseSensitive(config_json, "baseline"), &ns->ipd); - ns->ipd = ns->ipd / 1000.0f; // converts from mm to m - /*! @todo (Moses Turner) Next four u_json_get_float_array calls - * don't make any sense. They put the X coefficients from the - * JSON file into the Y coefficients in the structs, which is - * totally wrong, but the distortion looks totally wrong if we - * don't do this. - */ - u_json_get_float_array(cJSON_GetObjectItemCaseSensitive(config_json, "left_uv_to_rect_x"), - ns->eye_configs_v2[0].y_coefficients, 16); - u_json_get_float_array(cJSON_GetObjectItemCaseSensitive(config_json, "left_uv_to_rect_y"), - ns->eye_configs_v2[0].x_coefficients, 16); - u_json_get_float_array(cJSON_GetObjectItemCaseSensitive(config_json, "right_uv_to_rect_x"), - ns->eye_configs_v2[1].y_coefficients, 16); - u_json_get_float_array(cJSON_GetObjectItemCaseSensitive(config_json, "right_uv_to_rect_y"), - ns->eye_configs_v2[1].x_coefficients, 16); - bool said_first_thing = false; - if (!u_json_get_float(cJSON_GetObjectItemCaseSensitive(config_json, "left_fov_radians_left"), - &ns->eye_configs_v2[0].fov.angle_left)) { // not putting this directly in - // (&ns->base.hmd->views[eye_index].fov - // because i smell a rat there - - // that value seems to unexpectedly - // change during init process. - NS_INFO(ns, - "Just so you know, you can add tunable FoV parameters to your v2 json file. There are " - "examples in src/xrt/drivers/north_star/exampleconfigs.\n"); - said_first_thing = true; - ns->eye_configs_v2[0].fov.angle_left = -0.8; - ns->eye_configs_v2[0].fov.angle_right = 0.8; - ns->eye_configs_v2[0].fov.angle_up = 0.8; - ns->eye_configs_v2[0].fov.angle_down = -0.8; - - ns->eye_configs_v2[1].fov.angle_left = -0.8; - ns->eye_configs_v2[1].fov.angle_right = 0.8; - ns->eye_configs_v2[1].fov.angle_up = 0.8; - ns->eye_configs_v2[1].fov.angle_down = -0.8; - - - - } else { - - /*! @todo (Moses Turner) Something's wrong with either - * ns_v2_mesh_calc or this code, because when you have - * uneven FOV bounds, the distortion looks totally - * wrong.*/ - u_json_get_float(cJSON_GetObjectItemCaseSensitive(config_json, "left_fov_radians_left"), - &ns->eye_configs_v2[0].fov.angle_left); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(config_json, "left_fov_radians_right"), - &ns->eye_configs_v2[0].fov.angle_right); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(config_json, "left_fov_radians_up"), - &ns->eye_configs_v2[0].fov.angle_up); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(config_json, "left_fov_radians_down"), - &ns->eye_configs_v2[0].fov.angle_down); - - u_json_get_float(cJSON_GetObjectItemCaseSensitive(config_json, "right_fov_radians_left"), - &ns->eye_configs_v2[1].fov.angle_left); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(config_json, "right_fov_radians_right"), - &ns->eye_configs_v2[1].fov.angle_right); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(config_json, "right_fov_radians_up"), - &ns->eye_configs_v2[1].fov.angle_up); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(config_json, "right_fov_radians_down"), - &ns->eye_configs_v2[1].fov.angle_down); - } - - struct cJSON *offset = cJSON_GetObjectItemCaseSensitive(config_json, "t265_to_nose_bridge"); - if (offset == NULL) { - if (said_first_thing) { - NS_INFO(ns, - "Also, you should put an offset parameter into the json file to transform your " - "head pose from the realsense to your nose bridge. There are some examples in " - "src/xrt/drivers/north_star/exampleconfigs/"); - } else { - NS_INFO(ns, - "You should put an offset parameter into the json file to transform your head " - "pose from the realsense to your nose bridge. There are some examples in " - "src/xrt/drivers/north_star/exampleconfigs/."); - } - } else { - struct cJSON *translation_meters = - cJSON_GetObjectItemCaseSensitive(offset, "translation_meters"); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(translation_meters, "x"), - &t265_to_nose_bridge.position.x); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(translation_meters, "y"), - &t265_to_nose_bridge.position.y); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(translation_meters, "z"), - &t265_to_nose_bridge.position.z); - - struct cJSON *rotation_quaternion = - cJSON_GetObjectItemCaseSensitive(offset, "rotation_quaternion"); - - u_json_get_float(cJSON_GetObjectItemCaseSensitive(rotation_quaternion, "x"), - &t265_to_nose_bridge.orientation.x); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(rotation_quaternion, "y"), - &t265_to_nose_bridge.orientation.y); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(rotation_quaternion, "z"), - &t265_to_nose_bridge.orientation.z); - u_json_get_float(cJSON_GetObjectItemCaseSensitive(rotation_quaternion, "w"), - &t265_to_nose_bridge.orientation.w); - } - - ns->is_v2 = true; - - - } else if (cJSON_GetObjectItemCaseSensitive(config_json, "leftEye") != NULL && - cJSON_GetObjectItemCaseSensitive(config_json, "left_uv_to_rect_x") == NULL) { - ns_eye_parse(&ns->eye_configs_v1[0], cJSON_GetObjectItemCaseSensitive(config_json, "leftEye")); - - ns_eye_parse(&ns->eye_configs_v1[1], cJSON_GetObjectItemCaseSensitive(config_json, "rightEye")); - ns_leap_parse(&ns->leap_config, cJSON_GetObjectItemCaseSensitive(config_json, "leapTracker")); - ns->is_v2 = false; - } else { - NS_ERROR(ns, - "Bad config file. There are examples of v1 and v2 files in " - "src/xrt/drivers/north_star/exampleconfigs - if those don't work, something's really wrong."); + goto parse_error; } - cJSON_Delete(config_json); + // this function is not supposed to return true if ns->config_json is NULL + assert(ns->config_json != NULL); + free(json); + return true; + +parse_error: + if (json_allocated) { + free(json); + } + NS_INFO(ns, "Are you sure you're using the right configuration file?"); + return false; } @@ -487,97 +539,103 @@ ns_hmd_create(const char *config_path) { enum u_device_alloc_flags flags = (enum u_device_alloc_flags)(U_DEVICE_ALLOC_HMD | U_DEVICE_ALLOC_TRACKING_NONE); - - struct ns_hmd *ns = U_DEVICE_ALLOCATE(struct ns_hmd, flags, 1, 0); + ns->ll = debug_get_log_option_ns_log(); + + if (!ns_config_load(ns, config_path)) + goto cleanup; // don't need to print any error, ns_config_load did that for us + + int number_wrap = 3; // number of elements in below array of function pointers. Const to stop compiler warnings. + bool (*wrap_func_ptr[3])(struct ns_hmd *) = {ns_3d_parse, ns_p2d_parse, ns_vipd_parse}; + // C syntax is weird here. This is an array of pointers to functions with arguments (struct ns_system * system) + // that all return a boolean value. The array should be roughly in descending order of how likely we think the + // user means to use each method For now VIPD is last because Moses is the only one that uses it + + bool found_config_wrap = false; + for (int i = 0; i < number_wrap; i++) { + if (wrap_func_ptr[i](ns)) { // wrap_func_ptr[i](ns) is a function call! + U_LOG_I("North Star: Using config wrap %i", i); + found_config_wrap = true; + break; + } + } // This will segfault at function ?? if you use GDB and the length is wrong. + + if (!found_config_wrap) { + NS_INFO(ns, "North Star: Config file seems to be invalid."); + goto cleanup; + } + ns->base.update_inputs = ns_hmd_update_inputs; ns->base.get_tracked_pose = ns_hmd_get_tracked_pose; - // NOT HERE ns->base.get_view_pose = ns_hmd_get_view_pose; + ns->base.get_view_pose = ns_hmd_get_view_pose; ns->base.destroy = ns_hmd_destroy; ns->base.name = XRT_DEVICE_GENERIC_HMD; - ns->pose.orientation.w = 1.0f; // All other values set to zero. - ns->config_path = config_path; - ns->ll = debug_get_log_option_ns_log(); + math_pose_identity(&ns->no_tracker_relation.pose); + ns->no_tracker_relation.relation_flags = (enum xrt_space_relation_flags)( + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_POSITION_VALID_BIT | + XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT); + // Appeases the inner workings of Monado for when there's no head tracker and we're giving a fake pose through + // the debug gui + ns->base.orientation_tracking_supported = true; + ns->base.position_tracking_supported = true; + ns->base.device_type = XRT_DEVICE_TYPE_HMD; + // Print name. snprintf(ns->base.str, XRT_DEVICE_NAME_LEN, "North Star"); + snprintf(ns->base.serial, XRT_DEVICE_NAME_LEN, "North Star"); // Setup input. ns->base.inputs[0].name = XRT_INPUT_GENERIC_HEAD_POSE; - // Setup info. - struct u_device_simple_info info; - info.display.w_pixels = 2880; - info.display.h_pixels = 1440; - info.display.w_meters = 0.13f; - info.display.h_meters = 0.07f; - info.lens_horizontal_separation_meters = 0.13f / 2.0f; - info.lens_vertical_position_meters = 0.07f / 2.0f; - info.views[0].fov = 70.0f * (M_PI / 180.0f); - info.views[1].fov = 70.0f * (M_PI / 180.0f); + struct u_extents_2d exts; + // info.w_meters = 0.0588f * 2.0f; + // info.h_meters = 0.0655f; - if (!ns_config_load(ns)) - goto cleanup; + // one NS screen is 1440px wide, but there are two of them. + exts.w_pixels = 1440 * 2; + // Both NS screens are 1600px tall + exts.h_pixels = 1600; - // ns_config_load() sets ns->is_v2. - // to change how we switch between v1 and v2, - // you'll need to start in ns_config_load() - if (ns->is_v2) { - ns->base.get_view_pose = ns_v2_hmd_get_view_pose; - ns_v2_fov_calculate(ns, 0); - ns_v2_fov_calculate(ns, 1); - // Setup the north star basic info - if (!u_device_setup_split_side_by_side(&ns->base, &info)) { - NS_ERROR(ns, "Failed to setup basic device info"); - goto cleanup; - } - ns_v2_fov_calculate(ns, 0); - ns_v2_fov_calculate(ns, 1); + u_extents_2d_split_side_by_side(&ns->base, &exts); - ns->base.hmd->distortion.models = XRT_DISTORTION_MODEL_COMPUTE; - ns->base.hmd->distortion.preferred = XRT_DISTORTION_MODEL_COMPUTE; - ns->base.compute_distortion = ns_v2_mesh_calc; + ns->base.hmd->distortion.models = XRT_DISTORTION_MODEL_COMPUTE; + ns->base.hmd->distortion.preferred = XRT_DISTORTION_MODEL_COMPUTE; - } else { - // V1 - ns->base.get_view_pose = ns_hmd_get_view_pose; - ns_fov_calculate(&ns->base.hmd->views[0].fov, ns->eye_configs_v1[0].camera_projection); - ns_fov_calculate(&ns->base.hmd->views[1].fov, ns->eye_configs_v1[1].camera_projection); + ns->base.get_view_pose = ns_hmd_get_view_pose; - // Create the optical systems - ns->eye_configs_v1[0].optical_system = ns_create_optical_system(&ns->eye_configs_v1[0]); - ns->eye_configs_v1[1].optical_system = ns_create_optical_system(&ns->eye_configs_v1[1]); - - // Setup the north star basic info - if (!u_device_setup_split_side_by_side(&ns->base, &info)) { - NS_ERROR(ns, "Failed to setup basic device info"); - goto cleanup; - } - - ns->base.hmd->distortion.models = XRT_DISTORTION_MODEL_COMPUTE; - ns->base.hmd->distortion.preferred = XRT_DISTORTION_MODEL_COMPUTE; - ns->base.compute_distortion = ns_mesh_calc; - } - - // If built, try to load the realsense tracker. -#ifdef XRT_BUILD_DRIVER_RS - ns->tracker = rs_6dof_create(); - if (ns->tracker == NULL) { - NS_ERROR(ns, "Couldn't create realsense device!"); - } else { - rs_update_offset(t265_to_nose_bridge, ns->tracker); - } -#endif // Setup variable tracker. u_var_add_root(ns, "North Star", true); - u_var_add_pose(ns, &ns->pose, "pose"); + u_var_add_pose(ns, &ns->no_tracker_relation.pose, "pose"); ns->base.orientation_tracking_supported = true; - ns->base.position_tracking_supported = ns->tracker != NULL; - if (ns->tracker) { - ns->base.tracking_origin->type = ns->tracker->tracking_origin->type; - } ns->base.device_type = XRT_DEVICE_TYPE_HMD; + size_t idx = 0; + // Preferred; almost all North Stars (as of early 2021) are see-through. + ns->base.hmd->blend_modes[idx++] = XRT_BLEND_MODE_ADDITIVE; + + // XRT_BLEND_MODE_OPAQUE is not preferred and kind of a lie, but you can totally use North Star for VR apps, + // despite its see-through display. And there's nothing stopping you from covering up the outside of the + // reflector, turning it into an opaque headset. As most VR apps I've encountered require BLEND_MODE_OPAQUE to + // be an option, we need to support it. + ns->base.hmd->blend_modes[idx++] = XRT_BLEND_MODE_OPAQUE; + + // Not supporting ALPHA_BLEND for now, because I know nothing about it, have no reason to use it, and want to + // avoid unintended consequences. As soon as you have a specific reason to support it, go ahead and support it. + ns->base.hmd->num_blend_modes = idx; + + uint64_t start, end; + + start = os_monotonic_get_ns(); + u_distortion_mesh_fill_in_compute(&ns->base); + end = os_monotonic_get_ns(); + + float diff = (end - start); + diff /= U_TIME_1MS_IN_NS; + + NS_DEBUG(ns, "Filling mesh took %f ms", diff); + + return &ns->base; cleanup: diff --git a/src/xrt/drivers/north_star/ns_hmd.h b/src/xrt/drivers/north_star/ns_hmd.h index 4fe42da68..5065ea128 100644 --- a/src/xrt/drivers/north_star/ns_hmd.h +++ b/src/xrt/drivers/north_star/ns_hmd.h @@ -1,9 +1,12 @@ // Copyright 2019, Collabora, Ltd. +// Copyright 2020, Nova King. +// Copyright 2021, Moses Turner. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Interface between North Star distortion and HMD code. * @author Nova King + * @author Moses Turner * @ingroup drv_ns */ @@ -15,6 +18,10 @@ #include "xrt/xrt_defines.h" #include "xrt/xrt_device.h" #include "util/u_logging.h" +#include "os/os_threading.h" +#include "util/u_distortion_mesh.h" + +#include #ifdef __cplusplus extern "C" { @@ -34,7 +41,8 @@ extern "C" { /* * - * Structs + * 3D distortion structs + * Sometimes known as "v1", config file name is often "Calibration.json" * */ @@ -43,18 +51,7 @@ extern "C" { * * @ingroup drv_ns */ -struct ns_optical_system; - -/*! - * Simple UV struct. - * - * @ingroup drv_ns - */ -struct ns_uv -{ - float u; - float v; -}; +struct ns_3d_optical_system; /*! * Configuration information about the LMC or Rigel sensor according to the @@ -62,10 +59,10 @@ struct ns_uv * * @ingroup drv_ns */ -struct ns_leap +struct ns_3d_leap { - const char *name; - const char *serial; + char name[64]; + char serial[64]; struct xrt_pose pose; }; @@ -74,7 +71,7 @@ struct ns_leap * * @ingroup drv_ns */ -struct ns_v1_eye +struct ns_3d_eye { float ellipse_minor_axis; float ellipse_major_axis; @@ -92,12 +89,10 @@ struct ns_v1_eye struct ns_optical_system *optical_system; }; -struct ns_v2_eye +struct ns_3d_data { - float x_coefficients[16]; - float y_coefficients[16]; - struct xrt_pose eye_pose; - struct xrt_fov fov; + struct ns_3d_eye eyes[2]; + struct ns_3d_leap leap; }; /*! @@ -109,23 +104,19 @@ struct ns_v2_eye struct ns_hmd { - struct xrt_device base; - struct xrt_pose pose; - + struct xrt_space_relation no_tracker_relation; const char *config_path; + struct cJSON *config_json; + struct xrt_pose head_pose_to_eye[2]; // left, right - struct ns_v1_eye eye_configs_v1[2]; // will be NULL if is_v2. - struct ns_v2_eye eye_configs_v2[2]; // will be NULL if !is_v2 - float ipd; - - struct ns_leap leap_config; // will be NULL if is_v2 - - struct xrt_device *tracker; + union { + struct ns_3d_data dist_3d; + struct u_ns_p2d_values dist_p2d; + struct u_ns_vipd_values dist_vipd; + }; enum u_logging_level ll; - bool is_v2; // True if V2, false if V1. If we ever get a v3 this should - // be an enum or something }; /* @@ -145,16 +136,27 @@ ns_hmd(struct xrt_device *xdev) return (struct ns_hmd *)xdev; } + +/*! + * Create a North Star hmd. + * + * @ingroup drv_ns + */ +struct xrt_device * +ns_hmd_create(const char *config_path); + + /*! * Convert the display UV to the render UV using the distortion mesh. * * @ingroup drv_ns */ + void -ns_display_uv_to_render_uv(struct ns_uv in, struct ns_uv *out, struct ns_v1_eye *eye); +ns_3d_display_uv_to_render_uv(struct xrt_vec2 in, struct xrt_vec2 *out, struct ns_3d_eye *eye); struct ns_optical_system * -ns_create_optical_system(struct ns_v1_eye *eye); +ns_3d_create_optical_system(struct ns_3d_eye *eye); #ifdef __cplusplus diff --git a/src/xrt/drivers/north_star/ns_interface.h b/src/xrt/drivers/north_star/ns_interface.h index 3e878f212..4ccfb255d 100644 --- a/src/xrt/drivers/north_star/ns_interface.h +++ b/src/xrt/drivers/north_star/ns_interface.h @@ -30,14 +30,6 @@ extern "C" { struct xrt_auto_prober * ns_create_auto_prober(void); -/*! - * Create a North Star hmd. - * - * @ingroup drv_ns - */ -struct xrt_device * -ns_hmd_create(const char *config_path); - /*! * @dir drivers/north_star * diff --git a/src/xrt/drivers/north_star/ns_prober.c b/src/xrt/drivers/north_star/ns_prober.c index ff380f31b..bcd8383a1 100644 --- a/src/xrt/drivers/north_star/ns_prober.c +++ b/src/xrt/drivers/north_star/ns_prober.c @@ -18,6 +18,7 @@ #include "util/u_debug.h" #include "ns_interface.h" +#include "ns_hmd.h" DEBUG_GET_ONCE_OPTION(ns_config_path, "NS_CONFIG_PATH", NULL) @@ -48,26 +49,32 @@ ns_prober_destroy(struct xrt_auto_prober *p) } //! @public @memberof ns_prober -static struct xrt_device * -ns_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no_hmds, struct xrt_prober *xp) +static int +ns_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) { struct ns_prober *nsp = ns_prober(xap); if (no_hmds) { - return NULL; + return 0; } if (nsp->config_path == NULL) { - return NULL; + return 0; } - return ns_hmd_create(nsp->config_path); + out_xdevs[0] = ns_hmd_create(nsp->config_path); + return 1; } struct xrt_auto_prober * ns_create_auto_prober() { struct ns_prober *nsp = U_TYPED_CALLOC(struct ns_prober); + nsp->base.name = "northstar"; nsp->base.destroy = ns_prober_destroy; nsp->base.lelo_dallas_autoprobe = ns_prober_autoprobe; nsp->config_path = debug_get_option_ns_config_path(); diff --git a/src/xrt/drivers/ohmd/oh_device.c b/src/xrt/drivers/ohmd/oh_device.c index 9c1f93a18..fc6144802 100644 --- a/src/xrt/drivers/ohmd/oh_device.c +++ b/src/xrt/drivers/ohmd/oh_device.c @@ -10,6 +10,7 @@ #include "math/m_mathinclude.h" #include "xrt/xrt_config_os.h" +#include "xrt/xrt_prober.h" #include #include @@ -40,10 +41,63 @@ // directly retrieved? DEBUG_GET_ONCE_BOOL_OPTION(ohmd_finite_diff, "OHMD_ALLOW_FINITE_DIFF", true) DEBUG_GET_ONCE_LOG_OPTION(ohmd_log, "OHMD_LOG", U_LOGGING_WARN) +DEBUG_GET_ONCE_BOOL_OPTION(ohmd_external, "OHMD_EXTERNAL_DRIVER", false) // Define this if you have the appropriately hacked-up OpenHMD version. #undef OHMD_HAVE_ANG_VEL +enum input_indices +{ + // khronos simple inputs for generic controllers + SIMPLE_SELECT_CLICK = 0, + SIMPLE_MENU_CLICK, + SIMPLE_GRIP_POSE, + SIMPLE_AIM_POSE, + + // longest list of aliased enums has to start last for INPUT_INDICES_LAST to get the biggest value + OCULUS_TOUCH_X_CLICK = 0, + OCULUS_TOUCH_X_TOUCH, + OCULUS_TOUCH_Y_CLICK, + OCULUS_TOUCH_Y_TOUCH, + OCULUS_TOUCH_MENU_CLICK, + OCULUS_TOUCH_A_CLICK, + OCULUS_TOUCH_A_TOUCH, + OCULUS_TOUCH_B_CLICK, + OCULUS_TOUCH_B_TOUCH, + OCULUS_TOUCH_SYSTEM_CLICK, + OCULUS_TOUCH_SQUEEZE_VALUE, + OCULUS_TOUCH_TRIGGER_TOUCH, + OCULUS_TOUCH_TRIGGER_VALUE, + OCULUS_TOUCH_THUMBSTICK_CLICK, + OCULUS_TOUCH_THUMBSTICK_TOUCH, + OCULUS_TOUCH_THUMBSTICK, + OCULUS_TOUCH_THUMBREST_TOUCH, + OCULUS_TOUCH_GRIP_POSE, + OCULUS_TOUCH_AIM_POSE, + + INPUT_INDICES_LAST +}; +#define SET_TOUCH_INPUT(NAME) (ohd->base.inputs[OCULUS_TOUCH_##NAME].name = XRT_INPUT_TOUCH_##NAME) + +// all OpenHMD controls are floats, even "digital" controls. +// this means a trigger can be fed by a discrete button or by a pullable [0,1] analog trigger. +// in case of not so good calibrated triggers we may not reach 1.0 +#define PRESS_FLOAT_THRESHOLD 0.95 +#define FLOAT_TO_DIGITAL_THRESHOLD 0.5 + +// one mapping for each of enum ohmd_control_hint +#define CONTROL_MAPPING_SIZE 16 + +// generic controllers are mapped to the khronos simple profile +// touch controllers input mappings are special cased +enum openhmd_device_type +{ + OPENHMD_GENERIC_HMD, + OPENHMD_GENERIC_CONTROLLER, + OPENHMD_OCULUS_RIFT_HMD, + OPENHMD_OCULUS_RIFT_CONTROLLER, +}; + struct openhmd_values { float hmd_warp_param[4]; @@ -53,6 +107,18 @@ struct openhmd_values float warp_scale; }; +struct oh_device; +struct oh_system +{ + struct xrt_tracking_origin base; + struct oh_device *devices[XRT_MAX_DEVICES_PER_PROBE]; + + //! index into oh_system::devices + int hmd_idx; + int left_idx; + int right_idx; +}; + /*! * @implements xrt_device */ @@ -75,6 +141,27 @@ struct oh_device struct u_vive_values vive[2]; struct openhmd_values openhmd[2]; } distortion; + + struct oh_system *sys; + + + enum openhmd_device_type ohmd_device_type; + + // what function controls serve. + int controls_fn[64]; + + // whether controls are digital or analog. + // unused because the OpenXR interaction profile forces the type. + int controls_types[64]; + + //! maps OpenHMD control hint enum to the corresponding index into base.inputs + enum input_indices controls_mapping[CONTROL_MAPPING_SIZE]; + + // For touch controller we map an analog trigger to a float interaction profile input. + // For simple controller we map a potentially analog trigger to a bool input. + bool make_trigger_digital; + + float last_control_state[256]; }; static inline struct oh_device * @@ -88,21 +175,154 @@ oh_device_destroy(struct xrt_device *xdev) { struct oh_device *ohd = oh_device(xdev); - // Remove the variable tracking. - u_var_remove_root(ohd); - if (ohd->dev != NULL) { ohmd_close_device(ohd->dev); ohd->dev = NULL; } + bool all_null = true; + for (int i = 0; i < XRT_MAX_DEVICES_PER_PROBE; i++) { + if (ohd->sys->devices[i] == ohd) { + ohd->sys->devices[i] = NULL; + } + + if (ohd->sys->devices[i] != NULL) { + all_null = false; + } + } + + if (all_null) { + // Remove the variable tracking. + u_var_remove_root(ohd->sys); + + free(ohd->sys); + } + u_device_free(&ohd->base); } +#define CASE_VEC1(OHMD_CONTROL) \ + case OHMD_CONTROL: \ + if (ohd->controls_mapping[OHMD_CONTROL] == 0) { \ + break; \ + } \ + if (control_state[i] != ohd->last_control_state[i]) { \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].value.vec1.x = control_state[i]; \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].timestamp = ts; \ + } \ + break; + +#define CASE_VEC1_OR_DIGITAL(OHMD_CONTROL, MAKE_DIGITAL) \ + case OHMD_CONTROL: \ + if (ohd->controls_mapping[OHMD_CONTROL] == 0) { \ + break; \ + } \ + if (MAKE_DIGITAL) { \ + if ((control_state[i] > FLOAT_TO_DIGITAL_THRESHOLD) != \ + (ohd->last_control_state[i] > FLOAT_TO_DIGITAL_THRESHOLD)) { \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].value.vec1.x = \ + control_state[i] > FLOAT_TO_DIGITAL_THRESHOLD; \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].timestamp = ts; \ + } \ + } else { \ + if (control_state[i] != ohd->last_control_state[i]) { \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].value.vec1.x = control_state[i]; \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].timestamp = ts; \ + } \ + } \ + break; + +#define CASE_DIGITAL(OHMD_CONTROL, THRESHOLD) \ + case OHMD_CONTROL: \ + if (ohd->controls_mapping[OHMD_CONTROL] == 0) { \ + break; \ + } \ + if (control_state[i] != ohd->last_control_state[i]) { \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].value.boolean = \ + control_state[i] > THRESHOLD; \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].timestamp = ts; \ + } \ + break; + +#define CASE_VEC2_X(OHMD_CONTROL) \ + case OHMD_CONTROL: \ + if (ohd->controls_mapping[OHMD_CONTROL] == 0) { \ + break; \ + } \ + if (control_state[i] != ohd->last_control_state[i]) { \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].value.vec2.x = control_state[i]; \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].timestamp = ts; \ + } \ + break; + +#define CASE_VEC2_Y(OHMD_CONTROL) \ + case OHMD_CONTROL: \ + if (ohd->controls_mapping[OHMD_CONTROL] == 0) { \ + break; \ + } \ + if (control_state[i] != ohd->last_control_state[i]) { \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].value.vec2.y = control_state[i]; \ + ohd->base.inputs[ohd->controls_mapping[OHMD_CONTROL]].timestamp = ts; \ + } \ + break; + +static void +update_ohmd_controller(struct oh_device *ohd, int control_count, float *control_state) +{ + timepoint_ns ts = os_monotonic_get_ns(); + for (int i = 0; i < control_count; i++) { + switch (ohd->controls_fn[i]) { + // CASE macro does nothing, if ohd->controls_mapping[OHMD_CONTROL] has not been assigned + CASE_VEC1_OR_DIGITAL(OHMD_TRIGGER, ohd->make_trigger_digital); + CASE_DIGITAL(OHMD_TRIGGER_CLICK, PRESS_FLOAT_THRESHOLD); + CASE_VEC1(OHMD_SQUEEZE); + CASE_DIGITAL(OHMD_MENU, PRESS_FLOAT_THRESHOLD); + CASE_DIGITAL(OHMD_HOME, PRESS_FLOAT_THRESHOLD); + CASE_VEC2_X(OHMD_ANALOG_X); + CASE_VEC2_Y(OHMD_ANALOG_Y); + CASE_DIGITAL(OHMD_ANALOG_PRESS, PRESS_FLOAT_THRESHOLD); + CASE_DIGITAL(OHMD_BUTTON_A, PRESS_FLOAT_THRESHOLD); + CASE_DIGITAL(OHMD_BUTTON_B, PRESS_FLOAT_THRESHOLD); + CASE_DIGITAL(OHMD_BUTTON_X, PRESS_FLOAT_THRESHOLD); + CASE_DIGITAL(OHMD_BUTTON_Y, PRESS_FLOAT_THRESHOLD); + CASE_DIGITAL(OHMD_VOLUME_PLUS, PRESS_FLOAT_THRESHOLD); + CASE_DIGITAL(OHMD_VOLUME_MINUS, PRESS_FLOAT_THRESHOLD); + CASE_DIGITAL(OHMD_MIC_MUTE, PRESS_FLOAT_THRESHOLD); + } + } +} + static void oh_device_update_inputs(struct xrt_device *xdev) { - // Empty + struct oh_device *ohd = oh_device(xdev); + + int control_count; + float control_state[256]; + + ohmd_device_geti(ohd->dev, OHMD_CONTROL_COUNT, &control_count); + if (control_count > 64) + control_count = 64; + + ohmd_device_getf(ohd->dev, OHMD_CONTROLS_STATE, control_state); + + if (ohd->ohmd_device_type == OPENHMD_OCULUS_RIFT_CONTROLLER || + ohd->ohmd_device_type == OPENHMD_GENERIC_CONTROLLER) { + update_ohmd_controller(ohd, control_count, control_state); + } + + for (int i = 0; i < 256; i++) { + ohd->last_control_state[i] = control_state[i]; + } +} + +static void +oh_device_set_output(struct xrt_device *xdev, enum xrt_output_name name, union xrt_output_value *value) +{ + struct oh_device *ohd = oh_device(xdev); + (void)ohd; + + //! @todo OpenHMD haptic API not finished } static void @@ -112,10 +332,16 @@ oh_device_get_tracked_pose(struct xrt_device *xdev, struct xrt_space_relation *out_relation) { struct oh_device *ohd = oh_device(xdev); - struct xrt_quat quat = {0.f, 0.f, 0.f, 1.f}; - struct xrt_vec3 pos = {0.f, 0.f, 0.f}; + struct xrt_quat quat = XRT_QUAT_IDENTITY; + struct xrt_vec3 pos = XRT_VEC3_ZERO; - if (name != XRT_INPUT_GENERIC_HEAD_POSE) { + // support generic head pose for all hmds, + // support rift poses for rift controllers, and simple poses for generic controller + if (name != XRT_INPUT_GENERIC_HEAD_POSE && + (ohd->ohmd_device_type == OPENHMD_OCULUS_RIFT_CONTROLLER && + (name != XRT_INPUT_TOUCH_AIM_POSE && name != XRT_INPUT_TOUCH_GRIP_POSE)) && + ohd->ohmd_device_type == OPENHMD_GENERIC_CONTROLLER && + (name != XRT_INPUT_SIMPLE_AIM_POSE && name != XRT_INPUT_SIMPLE_GRIP_POSE)) { OHMD_ERROR(ohd, "unknown input name"); return; } @@ -160,7 +386,7 @@ oh_device_get_tracked_pose(struct xrt_device *xdev, * USB data and use that instead. */ *out_relation = ohd->last_relation; - OHMD_TRACE(ohd, "GET_TRACKED_POSE - no new data"); + OHMD_TRACE(ohd, "GET_TRACKED_POSE (%s) - no new data", ohd->base.str); return; } @@ -192,10 +418,11 @@ oh_device_get_tracked_pose(struct xrt_device *xdev, out_relation->relation_flags = (enum xrt_space_relation_flags)( out_relation->relation_flags | XRT_SPACE_RELATION_ANGULAR_VELOCITY_VALID_BIT); - OHMD_TRACE(ohd, "GET_TRACKED_POSE (%f, %f, %f, %f) (%f, %f, %f)", quat.x, quat.y, quat.z, quat.w, - ang_vel.x, ang_vel.y, ang_vel.z); + OHMD_TRACE(ohd, "GET_TRACKED_POSE (%s) (%f, %f, %f, %f) (%f, %f, %f)", ohd->base.str, quat.x, quat.y, + quat.z, quat.w, ang_vel.x, ang_vel.y, ang_vel.z); } else { - OHMD_TRACE(ohd, "GET_TRACKED_POSE (%f, %f, %f, %f)", quat.x, quat.y, quat.z, quat.w); + OHMD_TRACE(ohd, "GET_TRACKED_POSE (%s) (%f, %f, %f, %f)", ohd->base.str, quat.x, quat.y, quat.z, + quat.w); } // Update state within driver @@ -205,29 +432,12 @@ oh_device_get_tracked_pose(struct xrt_device *xdev, static void oh_device_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { - struct xrt_pose pose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; - bool adjust = view_index == 0; - - pose.position.x = eye_relation->x / 2.0f; - pose.position.y = eye_relation->y / 2.0f; - pose.position.z = eye_relation->z / 2.0f; - - // Adjust for left/right while also making sure there aren't any -0.f. - if (pose.position.x > 0.0f && adjust) { - pose.position.x = -pose.position.x; - } - if (pose.position.y > 0.0f && adjust) { - pose.position.y = -pose.position.y; - } - if (pose.position.z > 0.0f && adjust) { - pose.position.z = -pose.position.z; - } - - *out_pose = pose; + (void)xdev; + u_device_get_view_pose(eye_relation, view_index, out_pose); } struct display_info @@ -284,21 +494,21 @@ struct device_info }; static struct device_info -get_info(struct oh_device *ohd, const char *prod) +get_info(ohmd_device *dev, const char *prod) { struct device_info info = {0}; // clang-format off - ohmd_device_getf(ohd->dev, OHMD_SCREEN_HORIZONTAL_SIZE, &info.display.w_meters); - ohmd_device_getf(ohd->dev, OHMD_SCREEN_VERTICAL_SIZE, &info.display.h_meters); - ohmd_device_getf(ohd->dev, OHMD_LENS_HORIZONTAL_SEPARATION, &info.lens_horizontal_separation); - ohmd_device_getf(ohd->dev, OHMD_LENS_VERTICAL_POSITION, &info.lens_vertical_position); - ohmd_device_getf(ohd->dev, OHMD_LEFT_EYE_FOV, &info.views[0].fov); - ohmd_device_getf(ohd->dev, OHMD_RIGHT_EYE_FOV, &info.views[1].fov); - ohmd_device_geti(ohd->dev, OHMD_SCREEN_HORIZONTAL_RESOLUTION, &info.display.w_pixels); - ohmd_device_geti(ohd->dev, OHMD_SCREEN_VERTICAL_RESOLUTION, &info.display.h_pixels); - ohmd_device_getf(ohd->dev, OHMD_UNIVERSAL_DISTORTION_K, &info.pano_distortion_k[0]); - ohmd_device_getf(ohd->dev, OHMD_UNIVERSAL_ABERRATION_K, &info.pano_aberration_k[0]); + ohmd_device_getf(dev, OHMD_SCREEN_HORIZONTAL_SIZE, &info.display.w_meters); + ohmd_device_getf(dev, OHMD_SCREEN_VERTICAL_SIZE, &info.display.h_meters); + ohmd_device_getf(dev, OHMD_LENS_HORIZONTAL_SEPARATION, &info.lens_horizontal_separation); + ohmd_device_getf(dev, OHMD_LENS_VERTICAL_POSITION, &info.lens_vertical_position); + ohmd_device_getf(dev, OHMD_LEFT_EYE_FOV, &info.views[0].fov); + ohmd_device_getf(dev, OHMD_RIGHT_EYE_FOV, &info.views[1].fov); + ohmd_device_geti(dev, OHMD_SCREEN_HORIZONTAL_RESOLUTION, &info.display.w_pixels); + ohmd_device_geti(dev, OHMD_SCREEN_VERTICAL_RESOLUTION, &info.display.h_pixels); + ohmd_device_getf(dev, OHMD_UNIVERSAL_DISTORTION_K, &info.pano_distortion_k[0]); + ohmd_device_getf(dev, OHMD_UNIVERSAL_ABERRATION_K, &info.pano_aberration_k[0]); // Default to 90FPS info.display.nominal_frame_interval_ns = @@ -415,10 +625,10 @@ get_info(struct oh_device *ohd, const char *prod) if (info.quirks.rotate_screen_right_after) { // OpenHMD describes the logical orintation not the physical. // clang-format off - ohmd_device_getf(ohd->dev, OHMD_SCREEN_HORIZONTAL_SIZE, &info.display.h_meters); - ohmd_device_getf(ohd->dev, OHMD_SCREEN_VERTICAL_SIZE, &info.display.w_meters); - ohmd_device_geti(ohd->dev, OHMD_SCREEN_HORIZONTAL_RESOLUTION, &info.display.h_pixels); - ohmd_device_geti(ohd->dev, OHMD_SCREEN_VERTICAL_RESOLUTION, &info.display.w_pixels); + ohmd_device_getf(dev, OHMD_SCREEN_HORIZONTAL_SIZE, &info.display.h_meters); + ohmd_device_getf(dev, OHMD_SCREEN_VERTICAL_SIZE, &info.display.w_meters); + ohmd_device_geti(dev, OHMD_SCREEN_HORIZONTAL_RESOLUTION, &info.display.h_pixels); + ohmd_device_geti(dev, OHMD_SCREEN_VERTICAL_RESOLUTION, &info.display.w_pixels); // clang-format on } @@ -493,11 +703,19 @@ swap(int *a, int *b) *b = temp; } -struct xrt_device * -oh_device_create(ohmd_context *ctx, ohmd_device *dev, const char *prod) +static struct oh_device * +create_hmd(ohmd_context *ctx, int device_idx, int device_flags) { - enum u_device_alloc_flags flags = - (enum u_device_alloc_flags)(U_DEVICE_ALLOC_HMD | U_DEVICE_ALLOC_TRACKING_NONE); + const char *prod = ohmd_list_gets(ctx, device_idx, OHMD_PRODUCT); + ohmd_device *dev = ohmd_list_open_device(ctx, device_idx); + if (dev == NULL) { + return NULL; + } + + + const struct device_info info = get_info(dev, prod); + + enum u_device_alloc_flags flags = U_DEVICE_ALLOC_HMD; struct oh_device *ohd = U_DEVICE_ALLOCATE(struct oh_device, flags, 1, 0); ohd->base.update_inputs = oh_device_update_inputs; ohd->base.get_tracked_pose = oh_device_get_tracked_pose; @@ -509,10 +727,14 @@ oh_device_create(ohmd_context *ctx, ohmd_device *dev, const char *prod) ohd->dev = dev; ohd->ll = debug_get_log_option_ohmd_log(); ohd->enable_finite_difference = debug_get_bool_option_ohmd_finite_diff(); + if (strcmp(prod, "Rift (CV1)") == 0 || strcmp(prod, "Rift S") == 0) { + ohd->ohmd_device_type = OPENHMD_OCULUS_RIFT_HMD; + } else { + ohd->ohmd_device_type = OPENHMD_GENERIC_HMD; + } snprintf(ohd->base.str, XRT_DEVICE_NAME_LEN, "%s (OpenHMD)", prod); - - const struct device_info info = get_info(ohd, prod); + snprintf(ohd->base.serial, XRT_DEVICE_NAME_LEN, "%s (OpenHMD)", prod); { /* right eye */ @@ -562,20 +784,20 @@ oh_device_create(ohmd_context *ctx, ohmd_device *dev, const char *prod) ohd->base.hmd->views[1].rot = u_device_rotation_ident; OHMD_DEBUG(ohd, - "Display/viewport/offset before rotation %dx%d/%dx%d/%dx%d, " - "%dx%d/%dx%d/%dx%d", - ohd->base.hmd->views[0].display.w_pixels, - ohd->base.hmd->views[0].display.h_pixels, - ohd->base.hmd->views[0].viewport.w_pixels, - ohd->base.hmd->views[0].viewport.h_pixels, - ohd->base.hmd->views[0].viewport.x_pixels, - ohd->base.hmd->views[0].viewport.y_pixels, - ohd->base.hmd->views[1].display.w_pixels, - ohd->base.hmd->views[1].display.h_pixels, - ohd->base.hmd->views[1].viewport.w_pixels, - ohd->base.hmd->views[1].viewport.h_pixels, - ohd->base.hmd->views[0].viewport.x_pixels, - ohd->base.hmd->views[0].viewport.y_pixels); + "Display/viewport/offset before rotation %dx%d/%dx%d/%dx%d, " + "%dx%d/%dx%d/%dx%d", + ohd->base.hmd->views[0].display.w_pixels, + ohd->base.hmd->views[0].display.h_pixels, + ohd->base.hmd->views[0].viewport.w_pixels, + ohd->base.hmd->views[0].viewport.h_pixels, + ohd->base.hmd->views[0].viewport.x_pixels, + ohd->base.hmd->views[0].viewport.y_pixels, + ohd->base.hmd->views[1].display.w_pixels, + ohd->base.hmd->views[1].display.h_pixels, + ohd->base.hmd->views[1].viewport.w_pixels, + ohd->base.hmd->views[1].viewport.h_pixels, + ohd->base.hmd->views[0].viewport.x_pixels, + ohd->base.hmd->views[0].viewport.y_pixels); for (int view = 0; view < 2; view++) { ohd->distortion.openhmd[view].hmd_warp_param[0] = info.pano_distortion_k[0]; @@ -600,11 +822,13 @@ oh_device_create(ohmd_context *ctx, ohmd_device *dev, const char *prod) ohd->base.compute_distortion = compute_distortion_openhmd; // Which blend modes does the device support. - ohd->base.hmd->blend_mode = XRT_BLEND_MODE_OPAQUE; + + size_t bm_idx = 0; if (info.quirks.video_see_through) { - ohd->base.hmd->blend_mode = - (enum xrt_blend_mode)(ohd->base.hmd->blend_mode | XRT_BLEND_MODE_ALPHA_BLEND); + ohd->base.hmd->blend_modes[bm_idx++] = XRT_BLEND_MODE_ALPHA_BLEND; } + ohd->base.hmd->blend_modes[bm_idx++] = XRT_BLEND_MODE_OPAQUE; + ohd->base.hmd->num_blend_modes = bm_idx; if (info.quirks.video_distortion_vive) { // clang-format off @@ -779,8 +1003,279 @@ oh_device_create(ohmd_context *ctx, ohmd_device *dev, const char *prod) u_device_dump_config(&ohd->base, __func__, prod); } - u_var_add_root(ohd, "OpenHMD Wrapper", true); - u_var_add_ro_text(ohd, ohd->base.str, "Card"); + ohd->base.orientation_tracking_supported = (device_flags & OHMD_DEVICE_FLAGS_ROTATIONAL_TRACKING) != 0; + ohd->base.position_tracking_supported = (device_flags & OHMD_DEVICE_FLAGS_POSITIONAL_TRACKING) != 0; + ohd->base.device_type = XRT_DEVICE_TYPE_HMD; - return &ohd->base; + + if (info.quirks.delay_after_initialization) { + unsigned int time_to_sleep = 1; + do { + //! @todo convert to os_nanosleep + time_to_sleep = sleep(time_to_sleep); + } while (time_to_sleep); + } + + if (ohd->ll <= U_LOGGING_DEBUG) { + u_device_dump_config(&ohd->base, __func__, prod); + } + + return ohd; +} + +static struct oh_device * +create_controller(ohmd_context *ctx, int device_idx, int device_flags, enum xrt_device_type device_type) +{ + const char *prod = ohmd_list_gets(ctx, device_idx, OHMD_PRODUCT); + ohmd_device *dev = ohmd_list_open_device(ctx, device_idx); + if (dev == NULL) { + return 0; + } + + bool oculus_touch = false; + + // khronos simple controller has 4 inputs + int num_inputs = 4; + int num_outputs = 0; + + if (strcmp(prod, "Rift (CV1): Right Controller") == 0 || strcmp(prod, "Rift (CV1): Left Controller") == 0 || + strcmp(prod, "Rift S: Right Controller") == 0 || strcmp(prod, "Rift S: Left Controller") == 0) { + oculus_touch = true; + + num_inputs = INPUT_INDICES_LAST; + num_outputs = 1; + } + + enum u_device_alloc_flags flags = 0; + struct oh_device *ohd = U_DEVICE_ALLOCATE(struct oh_device, flags, num_inputs, num_outputs); + ohd->base.update_inputs = oh_device_update_inputs; + ohd->base.set_output = oh_device_set_output; + ohd->base.get_tracked_pose = oh_device_get_tracked_pose; + ohd->base.get_view_pose = oh_device_get_view_pose; + ohd->base.destroy = oh_device_destroy; + if (oculus_touch) { + ohd->ohmd_device_type = OPENHMD_OCULUS_RIFT_CONTROLLER; + ohd->base.name = XRT_DEVICE_TOUCH_CONTROLLER; + } else { + ohd->ohmd_device_type = OPENHMD_GENERIC_CONTROLLER; + ohd->base.name = XRT_DEVICE_GENERIC_HMD; //! @todo generic tracker + } + ohd->ctx = ctx; + ohd->dev = dev; + ohd->ll = debug_get_log_option_ohmd_log(); + ohd->enable_finite_difference = debug_get_bool_option_ohmd_finite_diff(); + + for (int i = 0; i < CONTROL_MAPPING_SIZE; i++) { + ohd->controls_mapping[i] = 0; + } + + if (oculus_touch) { + SET_TOUCH_INPUT(X_CLICK); + SET_TOUCH_INPUT(X_TOUCH); + SET_TOUCH_INPUT(Y_CLICK); + SET_TOUCH_INPUT(Y_TOUCH); + SET_TOUCH_INPUT(MENU_CLICK); + SET_TOUCH_INPUT(A_CLICK); + SET_TOUCH_INPUT(A_TOUCH); + SET_TOUCH_INPUT(B_CLICK); + SET_TOUCH_INPUT(B_TOUCH); + SET_TOUCH_INPUT(SYSTEM_CLICK); + SET_TOUCH_INPUT(SQUEEZE_VALUE); + SET_TOUCH_INPUT(TRIGGER_TOUCH); + SET_TOUCH_INPUT(TRIGGER_VALUE); + SET_TOUCH_INPUT(THUMBSTICK_CLICK); + SET_TOUCH_INPUT(THUMBSTICK_TOUCH); + SET_TOUCH_INPUT(THUMBSTICK); + SET_TOUCH_INPUT(THUMBREST_TOUCH); + SET_TOUCH_INPUT(GRIP_POSE); + SET_TOUCH_INPUT(AIM_POSE); + + ohd->make_trigger_digital = false; + + ohd->base.outputs[0].name = XRT_OUTPUT_NAME_TOUCH_HAPTIC; + + ohd->controls_mapping[OHMD_TRIGGER] = OCULUS_TOUCH_TRIGGER_VALUE; + ohd->controls_mapping[OHMD_SQUEEZE] = OCULUS_TOUCH_SQUEEZE_VALUE; + ohd->controls_mapping[OHMD_MENU] = OCULUS_TOUCH_MENU_CLICK; + ohd->controls_mapping[OHMD_HOME] = OCULUS_TOUCH_SYSTEM_CLICK; + ohd->controls_mapping[OHMD_ANALOG_X] = OCULUS_TOUCH_THUMBSTICK; + ohd->controls_mapping[OHMD_ANALOG_Y] = OCULUS_TOUCH_THUMBSTICK; + ohd->controls_mapping[OHMD_ANALOG_PRESS] = OCULUS_TOUCH_THUMBSTICK_CLICK; + ohd->controls_mapping[OHMD_BUTTON_A] = OCULUS_TOUCH_A_CLICK; + ohd->controls_mapping[OHMD_BUTTON_B] = OCULUS_TOUCH_B_CLICK; + ohd->controls_mapping[OHMD_BUTTON_X] = OCULUS_TOUCH_X_CLICK; + ohd->controls_mapping[OHMD_BUTTON_Y] = OCULUS_TOUCH_Y_CLICK; + } else { + ohd->base.inputs[SIMPLE_SELECT_CLICK].name = XRT_INPUT_SIMPLE_SELECT_CLICK; + ohd->base.inputs[SIMPLE_MENU_CLICK].name = XRT_INPUT_SIMPLE_MENU_CLICK; + ohd->base.inputs[SIMPLE_GRIP_POSE].name = XRT_INPUT_SIMPLE_GRIP_POSE; + ohd->base.inputs[SIMPLE_AIM_POSE].name = XRT_INPUT_SIMPLE_AIM_POSE; + + // XRT_INPUT_SIMPLE_SELECT_CLICK is digital input. + // in case the hardware is an analog trigger, change the input after a half pulled trigger. + ohd->make_trigger_digital = true; + + if (num_outputs > 0) { + ohd->base.outputs[0].name = XRT_OUTPUT_NAME_SIMPLE_VIBRATION; + } + + ohd->controls_mapping[OHMD_TRIGGER] = SIMPLE_SELECT_CLICK; + ohd->controls_mapping[OHMD_MENU] = SIMPLE_MENU_CLICK; + } + + snprintf(ohd->base.str, XRT_DEVICE_NAME_LEN, "%s (OpenHMD)", prod); + snprintf(ohd->base.serial, XRT_DEVICE_NAME_LEN, "%s (OpenHMD)", prod); + + ohd->base.orientation_tracking_supported = (device_flags & OHMD_DEVICE_FLAGS_ROTATIONAL_TRACKING) != 0; + ohd->base.position_tracking_supported = (device_flags & OHMD_DEVICE_FLAGS_POSITIONAL_TRACKING) != 0; + ohd->base.device_type = device_type; + + ohmd_device_geti(ohd->dev, OHMD_CONTROLS_HINTS, ohd->controls_fn); + ohmd_device_geti(ohd->dev, OHMD_CONTROLS_TYPES, ohd->controls_types); + + OHMD_DEBUG(ohd, "Created %s controller", + device_type == XRT_DEVICE_TYPE_LEFT_HAND_CONTROLLER ? "left" : "right"); + + return ohd; +} + +int +oh_device_create(ohmd_context *ctx, bool no_hmds, struct xrt_device **out_xdevs) +{ + int hmd_idx = -1; + int hmd_flags = 0; + int left_idx = -1; + int left_flags = 0; + int right_idx = -1; + int right_flags = 0; + + if (no_hmds) { + return 0; + } + + /* Probe for devices */ + int num_devices = ohmd_ctx_probe(ctx); + + /* Then loop */ + for (int i = 0; i < num_devices; i++) { + int device_class = 0, device_flags = 0; + const char *prod = NULL; + + ohmd_list_geti(ctx, i, OHMD_DEVICE_CLASS, &device_class); + ohmd_list_geti(ctx, i, OHMD_DEVICE_FLAGS, &device_flags); + + if (device_class == OHMD_DEVICE_CLASS_CONTROLLER) { + if ((device_flags & OHMD_DEVICE_FLAGS_LEFT_CONTROLLER) != 0) { + if (left_idx != -1) { + continue; + } + U_LOG_D("Selecting left controller idx %i", i); + left_idx = i; + left_flags = device_flags; + } + if ((device_flags & OHMD_DEVICE_FLAGS_RIGHT_CONTROLLER) != 0) { + if (right_idx != -1) { + continue; + } + U_LOG_D("Selecting right controller idx %i", i); + right_idx = i; + right_flags = device_flags; + } + + continue; + } + + if (device_class == OHMD_DEVICE_CLASS_HMD) { + if (hmd_idx != -1) { + continue; + } + U_LOG_D("Selecting hmd idx %i", i); + hmd_idx = i; + hmd_flags = device_flags; + } + + if (device_flags & OHMD_DEVICE_FLAGS_NULL_DEVICE) { + U_LOG_D("Rejecting device idx %i, is a NULL device.", i); + continue; + } + + prod = ohmd_list_gets(ctx, i, OHMD_PRODUCT); + if (strcmp(prod, "External Device") == 0 && !debug_get_bool_option_ohmd_external()) { + U_LOG_D("Rejecting device idx %i, is a External device.", i); + continue; + } + + if (hmd_idx != -1 && left_idx != -1 && right_idx != -1) { + break; + } + } + + + // All OpenHMD devices share the same tracking origin. + // Default everything to 3dof (NONE), but 6dof when the HMD supports position tracking. + //! @todo: support mix of 3dof and 6dof OpenHMD devices + struct oh_system *sys = U_TYPED_CALLOC(struct oh_system); + sys->base.type = XRT_TRACKING_TYPE_NONE; + sys->base.offset.orientation.w = 1.0f; + sys->hmd_idx = -1; + sys->left_idx = -1; + sys->right_idx = -1; + + u_var_add_root(sys, "OpenHMD Wrapper", false); + + int created = 0; + if (!no_hmds && hmd_idx != -1) { + struct oh_device *hmd = create_hmd(ctx, hmd_idx, hmd_flags); + if (hmd) { + hmd->sys = sys; + hmd->base.tracking_origin = &sys->base; + + sys->hmd_idx = created; + sys->devices[sys->hmd_idx] = hmd; + + if (hmd->base.position_tracking_supported) { + sys->base.type = XRT_TRACKING_TYPE_OTHER; + } + + out_xdevs[created++] = &hmd->base; + } + } + + if (left_idx != -1) { + struct oh_device *left = + create_controller(ctx, left_idx, left_flags, XRT_DEVICE_TYPE_LEFT_HAND_CONTROLLER); + if (left) { + left->sys = sys; + left->base.tracking_origin = &sys->base; + + sys->left_idx = created; + sys->devices[sys->left_idx] = left; + + out_xdevs[created++] = &left->base; + } + } + + if (right_idx != -1) { + struct oh_device *right = + create_controller(ctx, right_idx, right_flags, XRT_DEVICE_TYPE_RIGHT_HAND_CONTROLLER); + if (right) { + right->sys = sys; + right->base.tracking_origin = &sys->base; + + sys->right_idx = created; + sys->devices[sys->right_idx] = right; + + out_xdevs[created++] = &right->base; + } + } + + for (int i = 0; i < XRT_MAX_DEVICES_PER_PROBE; i++) { + if (sys->devices[i] != NULL) { + u_var_add_ro_text(sys, sys->devices[i]->base.str, "OpenHMD Device"); + } + } + + //! @todo initialize more devices like generic trackers (nolo) + + return created; } diff --git a/src/xrt/drivers/ohmd/oh_device.h b/src/xrt/drivers/ohmd/oh_device.h index 2d8794a19..633dc086b 100644 --- a/src/xrt/drivers/ohmd/oh_device.h +++ b/src/xrt/drivers/ohmd/oh_device.h @@ -16,8 +16,8 @@ extern "C" { #endif -struct xrt_device * -oh_device_create(ohmd_context *ctx, ohmd_device *dev, const char *prod); +int +oh_device_create(ohmd_context *ctx, bool no_hmds, struct xrt_device **out_xdevs); #define OHMD_TRACE(d, ...) U_LOG_XDEV_IFL_T(&d->base, d->ll, __VA_ARGS__) #define OHMD_DEBUG(d, ...) U_LOG_XDEV_IFL_D(&d->base, d->ll, __VA_ARGS__) diff --git a/src/xrt/drivers/ohmd/oh_prober.c b/src/xrt/drivers/ohmd/oh_prober.c index 3f0cfd878..726f08756 100644 --- a/src/xrt/drivers/ohmd/oh_prober.c +++ b/src/xrt/drivers/ohmd/oh_prober.c @@ -20,8 +20,6 @@ #include "oh_device.h" -DEBUG_GET_ONCE_BOOL_OPTION(ohmd_external, "OHMD_EXTERNAL_DRIVER", false) - /*! * @implements xrt_auto_prober */ @@ -53,72 +51,17 @@ oh_prober_destroy(struct xrt_auto_prober *p) } //! @public @memberof oh_prober -static struct xrt_device * -oh_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no_hmds, struct xrt_prober *xp) +static int +oh_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) { struct oh_prober *ohp = oh_prober(xap); - // Do not use OpenHMD if we are not looking for HMDs. - if (no_hmds) { - return NULL; - } - - int device_idx = -1; - - /* Probe for devices */ - int num_devices = ohmd_ctx_probe(ohp->ctx); - - bool orientation_tracking_supported = false; - bool position_tracking_supported = false; - /* Then loop */ - for (int i = 0; i < num_devices; i++) { - int device_class = 0, device_flags = 0; - const char *prod = NULL; - - ohmd_list_geti(ohp->ctx, i, OHMD_DEVICE_CLASS, &device_class); - ohmd_list_geti(ohp->ctx, i, OHMD_DEVICE_FLAGS, &device_flags); - - if (device_class != OHMD_DEVICE_CLASS_HMD) { - U_LOG_D("Rejecting device idx %i, is not a HMD.", i); - continue; - } - - if (device_flags & OHMD_DEVICE_FLAGS_NULL_DEVICE) { - U_LOG_D("Rejecting device idx %i, is a NULL device.", i); - continue; - } - - prod = ohmd_list_gets(ohp->ctx, i, OHMD_PRODUCT); - if (strcmp(prod, "External Device") == 0 && !debug_get_bool_option_ohmd_external()) { - U_LOG_D("Rejecting device idx %i, is a External device.", i); - continue; - } - - U_LOG_D("Selecting device idx %i", i); - device_idx = i; - - orientation_tracking_supported = (device_flags & OHMD_DEVICE_FLAGS_ROTATIONAL_TRACKING) != 0; - position_tracking_supported = (device_flags & OHMD_DEVICE_FLAGS_POSITIONAL_TRACKING) != 0; - break; - } - - if (device_idx < 0) { - return NULL; - } - - const char *prod = ohmd_list_gets(ohp->ctx, device_idx, OHMD_PRODUCT); - ohmd_device *dev = ohmd_list_open_device(ohp->ctx, device_idx); - if (dev == NULL) { - return NULL; - } - - struct xrt_device *xdev = oh_device_create(ohp->ctx, dev, prod); - - xdev->orientation_tracking_supported = orientation_tracking_supported; - xdev->position_tracking_supported = position_tracking_supported; - xdev->device_type = XRT_DEVICE_TYPE_HMD; - - return xdev; + int num_created = oh_device_create(ohp->ctx, no_hmds, out_xdevs); + return num_created; } struct xrt_auto_prober * diff --git a/src/xrt/drivers/psmv/psmv_driver.c b/src/xrt/drivers/psmv/psmv_driver.c index 96098dda3..75ebdc5da 100644 --- a/src/xrt/drivers/psmv/psmv_driver.c +++ b/src/xrt/drivers/psmv/psmv_driver.c @@ -37,7 +37,7 @@ /*! - * @ingroup drv_psmv + * @addtogroup drv_psmv * @{ */ @@ -1034,6 +1034,17 @@ psmv_found(struct xrt_prober *xp, psmv->log_level = debug_get_log_option_psmv_log(); psmv->pid = devices[index]->product_id; psmv->hid = hid; + + struct xrt_prober_device *dev = devices[index]; + int str_serial_ret = xrt_prober_get_string_descriptor(xp, dev, XRT_PROBER_STRING_SERIAL_NUMBER, + (unsigned char *)psmv->base.serial, XRT_DEVICE_NAME_LEN); + + static int controller_num = 0; + if (str_serial_ret <= 0) { + snprintf(psmv->base.serial, XRT_DEVICE_NAME_LEN, "PS Move Controller %d", controller_num++); + PSMV_ERROR(psmv, "Could not get bluetooth serial, fallback: %s", psmv->base.serial); + } + snprintf(psmv->base.str, XRT_DEVICE_NAME_LEN, "%s", "PS Move Controller"); m_imu_pre_filter_init(&psmv->calibration.prefilter, 1.f, 1.f); diff --git a/src/xrt/drivers/psvr/psvr_device.c b/src/xrt/drivers/psvr/psvr_device.c index 6918b4aaa..c7ba60b28 100644 --- a/src/xrt/drivers/psvr/psvr_device.c +++ b/src/xrt/drivers/psvr/psvr_device.c @@ -1,5 +1,5 @@ // Copyright 2016, Joey Ferwerda. -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -296,10 +296,7 @@ read_sample_and_apply_calibration(struct psvr_device *psvr, } static void -update_fusion(struct psvr_device *psvr, - struct psvr_parsed_sample *sample, - uint32_t tick_delta, - timepoint_ns timestamp_ns) +update_fusion(struct psvr_device *psvr, struct psvr_parsed_sample *sample, uint64_t timestamp_ns) { struct xrt_vec3 mag = {0.0f, 0.0f, 0.0f}; (void)mag; @@ -313,14 +310,7 @@ update_fusion(struct psvr_device *psvr, xrt_tracked_psvr_push_imu(psvr->tracker, timestamp_ns, &sample); } else { -#if 1 - timepoint_ns now = os_monotonic_get_ns(); - m_imu_3dof_update(&psvr->fusion, now, &psvr->read.accel, &psvr->read.gyro); -#else - float delta_secs = tick_delta / PSVR_TICKS_PER_SECOND; - - math_quat_integrate_velocity(&psvr->fusion.rot, &psvr->read.gyro, delta_secs, &psvr->fusion.rot); -#endif + m_imu_3dof_update(&psvr->fusion, timestamp_ns, &psvr->read.accel, &psvr->read.gyro); } } @@ -338,10 +328,27 @@ calc_delta_and_handle_rollover(uint32_t next, uint32_t last) return tick_delta; } +static timepoint_ns +ensure_forward_progress_timestamps(struct psvr_device *psvr, timepoint_ns timestamp_ns) +{ + timepoint_ns t = timestamp_ns; + + /* + * This make sure the timestamp is after the last we sent to the fusion, + * but it effectively drops the sample. + */ + if (psvr->last_sensor_time > t) { + t = psvr->last_sensor_time + 1; + } + + psvr->last_sensor_time = t; + return t; +} + static void handle_tracker_sensor_msg(struct psvr_device *psvr, unsigned char *buffer, int size) { - timepoint_ns now = os_monotonic_get_ns(); + timepoint_ns now_ns = os_monotonic_get_ns(); uint32_t last_sample_tick = psvr->last.samples[1].tick; if (!psvr_parse_sensor_packet(&psvr->last, buffer, size)) { @@ -368,16 +375,26 @@ handle_tracker_sensor_msg(struct psvr_device *psvr, unsigned char *buffer, int s tick_delta = 500; } } + // New delta between the two samples. uint32_t tick_delta2 = calc_delta_and_handle_rollover(s->samples[1].tick, s->samples[0].tick); time_duration_ns inter_sample_duration_ns = tick_delta2 * PSVR_NS_PER_TICK; + + // Move it back in time. + timepoint_ns timestamp_ns = (uint64_t)now_ns - (uint64_t)inter_sample_duration_ns; + + // Make sure timestamps are always after a previous timestamp. + timestamp_ns = ensure_forward_progress_timestamps(psvr, timestamp_ns); + // Update the fusion with first sample. - update_fusion(psvr, &s->samples[0], tick_delta, now - inter_sample_duration_ns); + update_fusion(psvr, &s->samples[0], timestamp_ns); + + // Make sure timestamps are always after a previous timestamp. + timestamp_ns = ensure_forward_progress_timestamps(psvr, now_ns); // Update the fusion with second sample. - update_fusion(psvr, &s->samples[1], tick_delta2, now); - psvr->last_sensor_time = now; + update_fusion(psvr, &s->samples[1], timestamp_ns); } static void @@ -912,29 +929,12 @@ psvr_device_get_tracked_pose(struct xrt_device *xdev, static void psvr_device_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { - struct xrt_pose pose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; - bool adjust = view_index == 0; - - pose.position.x = eye_relation->x / 2.0f; - pose.position.y = eye_relation->y / 2.0f; - pose.position.z = eye_relation->z / 2.0f; - - // Adjust for left/right while also making sure there aren't any -0.f. - if (pose.position.x > 0.0f && adjust) { - pose.position.x = -pose.position.x; - } - if (pose.position.y > 0.0f && adjust) { - pose.position.y = -pose.position.y; - } - if (pose.position.z > 0.0f && adjust) { - pose.position.z = -pose.position.z; - } - - *out_pose = pose; + (void)xdev; + u_device_get_view_pose(eye_relation, view_index, out_pose); } static void @@ -1012,6 +1012,7 @@ psvr_device_create(struct hid_device_info *sensor_hid_info, #endif snprintf(psvr->base.str, XRT_DEVICE_NAME_LEN, "PS VR Headset"); + snprintf(psvr->base.serial, XRT_DEVICE_NAME_LEN, "PS VR Headset"); ret = open_hid(psvr, sensor_hid_info, &psvr->hid_sensor); if (ret != 0) { diff --git a/src/xrt/drivers/psvr/psvr_prober.c b/src/xrt/drivers/psvr/psvr_prober.c index d61959afe..63eb86d79 100644 --- a/src/xrt/drivers/psvr/psvr_prober.c +++ b/src/xrt/drivers/psvr/psvr_prober.c @@ -74,8 +74,12 @@ psvr_prober_destroy(struct xrt_auto_prober *p) } //! @public @memberof psvr_prober -static struct xrt_device * -psvr_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no_hmds, struct xrt_prober *xp) +static int +psvr_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) { struct psvr_prober *ppsvr = psvr_prober(xap); struct hid_device_info *info_control = NULL; @@ -86,7 +90,7 @@ psvr_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no // Do not look for the PSVR if we are not looking for HMDs. if (no_hmds) { - return NULL; + return 0; } devs = hid_enumerate(PSVR_VID, PSVR_PID); @@ -110,7 +114,12 @@ psvr_prober_autoprobe(struct xrt_auto_prober *xap, cJSON *attached_data, bool no hid_free_enumeration(devs); - return dev; + if (!dev) { + return 0; + } + + out_xdevs[0] = dev; + return 1; } diff --git a/src/xrt/drivers/qwerty/qwerty_device.c b/src/xrt/drivers/qwerty/qwerty_device.c new file mode 100644 index 000000000..024eb2482 --- /dev/null +++ b/src/xrt/drivers/qwerty/qwerty_device.c @@ -0,0 +1,548 @@ +// Copyright 2021, Mateo de Mayo. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Implementation of qwerty_device related methods. + * @author Mateo de Mayo + * @ingroup drv_qwerty + */ + +#include "qwerty_device.h" + +#include "util/u_device.h" +#include "util/u_distortion_mesh.h" +#include "util/u_var.h" +#include "util/u_logging.h" + +#include "math/m_api.h" +#include "math/m_space.h" +#include "math/m_mathinclude.h" + +#include "xrt/xrt_device.h" + +#include +#include + +#define QWERTY_HMD_INITIAL_MOVEMENT_SPEED 0.002f // in meters per frame +#define QWERTY_HMD_INITIAL_LOOK_SPEED 0.02f // in radians per frame +#define QWERTY_CONTROLLER_INITIAL_MOVEMENT_SPEED 0.005f +#define QWERTY_CONTROLLER_INITIAL_LOOK_SPEED 0.05f +#define MOVEMENT_SPEED_STEP 1.25f // Multiplier for how fast will mov speed increase/decrease +#define SPRINT_STEPS 5 // Amount of MOVEMENT_SPEED_STEPs to increase when sprinting + +// clang-format off +// Values copied from u_device_setup_tracking_origins. CONTROLLER relative to HMD. +#define QWERTY_HMD_INITIAL_POS (struct xrt_vec3){0, 1.6f, 0} +#define QWERTY_CONTROLLER_INITIAL_POS(is_left) (struct xrt_vec3){(is_left) ? -0.2f : 0.2f, -0.3f, -0.5f} +// clang-format on + +// Indices for fake controller input components +#define QWERTY_SELECT 0 +#define QWERTY_MENU 1 +#define QWERTY_GRIP 2 +#define QWERTY_AIM 3 +#define QWERTY_VIBRATION 0 + +#define QWERTY_TRACE(qd, ...) U_LOG_XDEV_IFL_T(&qd->base, qd->sys->ll, __VA_ARGS__) +#define QWERTY_DEBUG(qd, ...) U_LOG_XDEV_IFL_D(&qd->base, qd->sys->ll, __VA_ARGS__) +#define QWERTY_INFO(qd, ...) U_LOG_XDEV_IFL_I(&qd->base, qd->sys->ll, __VA_ARGS__) +#define QWERTY_WARN(qd, ...) U_LOG_XDEV_IFL_W(&qd->base, qd->sys->ll, __VA_ARGS__) +#define QWERTY_ERROR(qd, ...) U_LOG_XDEV_IFL_E(&qd->base, qd->sys->ll, __VA_ARGS__) + +static void +qwerty_system_remove(struct qwerty_system *qs, struct qwerty_device *qd); + +static void +qwerty_system_destroy(struct qwerty_system *qs); + +// Compare any two pointers without verbose casts +static inline bool +eq(void *a, void *b) +{ + return a == b; +} + +// xrt_device functions + +struct qwerty_device * +qwerty_device(struct xrt_device *xd) +{ + struct qwerty_device *qd = (struct qwerty_device *)xd; + bool is_qwerty_device = eq(qd, qd->sys->hmd) || eq(qd, qd->sys->lctrl) || eq(qd, qd->sys->rctrl); + assert(is_qwerty_device); + if (!is_qwerty_device) { + return NULL; + } + return qd; +} + +struct qwerty_hmd * +qwerty_hmd(struct xrt_device *xd) +{ + struct qwerty_hmd *qh = (struct qwerty_hmd *)xd; + bool is_qwerty_hmd = eq(qh, qh->base.sys->hmd); + assert(is_qwerty_hmd); + if (!is_qwerty_hmd) { + return NULL; + } + return qh; +} + +struct qwerty_controller * +qwerty_controller(struct xrt_device *xd) +{ + struct qwerty_controller *qc = (struct qwerty_controller *)xd; + bool is_qwerty_controller = eq(qc, qc->base.sys->lctrl) || eq(qc, qc->base.sys->rctrl); + assert(is_qwerty_controller); + if (!is_qwerty_controller) { + return NULL; + } + return qc; +} + +static void +qwerty_update_inputs(struct xrt_device *xd) +{ + if (xd->name != XRT_DEVICE_SIMPLE_CONTROLLER) { + return; + } + + struct qwerty_controller *qc = qwerty_controller(xd); + struct qwerty_device *qd = &qc->base; + + xd->inputs[QWERTY_SELECT].value.boolean = qc->select_clicked; + if (qc->select_clicked) { + QWERTY_INFO(qd, "[%s] Select click", xd->str); + qc->select_clicked = false; + } + + xd->inputs[QWERTY_MENU].value.boolean = qc->menu_clicked; + if (qc->menu_clicked) { + QWERTY_INFO(qd, "[%s] Menu click", xd->str); + qc->menu_clicked = false; + } +} + +static void +qwerty_set_output(struct xrt_device *xd, enum xrt_output_name name, union xrt_output_value *value) +{ + struct qwerty_device *qd = qwerty_device(xd); + float frequency = value->vibration.frequency; + float amplitude = value->vibration.amplitude; + time_duration_ns duration = value->vibration.duration; + if (amplitude || duration || frequency) { + QWERTY_INFO(qd, + "[%s] Haptic output: \n" + "\tfrequency=%.2f amplitude=%.2f duration=%ld", + xd->str, frequency, amplitude, duration); + } +} + +static void +qwerty_get_tracked_pose(struct xrt_device *xd, + enum xrt_input_name name, + uint64_t at_timestamp_ns, + struct xrt_space_relation *out_relation) +{ + struct qwerty_device *qd = qwerty_device(xd); + + if (name != XRT_INPUT_GENERIC_HEAD_POSE && name != XRT_INPUT_SIMPLE_GRIP_POSE) { + QWERTY_ERROR(qd, "Unexpected input name = 0x%04X", name >> 8); + return; + } + + // Position + + float sprint_boost = qd->sprint_pressed ? powf(MOVEMENT_SPEED_STEP, SPRINT_STEPS) : 1; + float mov_speed = qd->movement_speed * sprint_boost; + struct xrt_vec3 pos_delta = { + mov_speed * (qd->right_pressed - qd->left_pressed), + 0, // Up/down movement will be relative to base space + mov_speed * (qd->backward_pressed - qd->forward_pressed), + }; + math_quat_rotate_vec3(&qd->pose.orientation, &pos_delta, &pos_delta); + pos_delta.y += mov_speed * (qd->up_pressed - qd->down_pressed); + math_vec3_accum(&pos_delta, &qd->pose.position); + + // Orientation + + // View rotation caused by keys + float y_look_speed = qd->look_speed * (qd->look_left_pressed - qd->look_right_pressed); + float x_look_speed = qd->look_speed * (qd->look_up_pressed - qd->look_down_pressed); + + // View rotation caused by mouse + y_look_speed += qd->yaw_delta; + x_look_speed += qd->pitch_delta; + qd->yaw_delta = 0; + qd->pitch_delta = 0; + + struct xrt_quat x_rotation, y_rotation; + const struct xrt_vec3 x_axis = XRT_VEC3_UNIT_X; + const struct xrt_vec3 y_axis = XRT_VEC3_UNIT_Y; + math_quat_from_angle_vector(x_look_speed, &x_axis, &x_rotation); + math_quat_from_angle_vector(y_look_speed, &y_axis, &y_rotation); + math_quat_rotate(&qd->pose.orientation, &x_rotation, &qd->pose.orientation); // local-space pitch + math_quat_rotate(&y_rotation, &qd->pose.orientation, &qd->pose.orientation); // base-space yaw + math_quat_normalize(&qd->pose.orientation); + + // HMD Parenting + + bool qd_is_ctrl = name == XRT_INPUT_SIMPLE_GRIP_POSE; + struct qwerty_controller *qc = qd_is_ctrl ? qwerty_controller(&qd->base) : NULL; + if (qd_is_ctrl && qc->follow_hmd) { + struct xrt_space_graph space_graph = {0}; + struct qwerty_device *qd_hmd = &qd->sys->hmd->base; + m_space_graph_add_pose(&space_graph, &qd->pose); // controller pose + m_space_graph_add_pose(&space_graph, &qd_hmd->pose); // base space is hmd space + m_space_graph_resolve(&space_graph, out_relation); + } else { + out_relation->pose = qd->pose; + } + out_relation->relation_flags = + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_POSITION_VALID_BIT | + XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT; +} + +static void +qwerty_get_view_pose(struct xrt_device *xdev, + const struct xrt_vec3 *eye_relation, + uint32_t view_index, + struct xrt_pose *out_pose) +{ + (void)xdev; + u_device_get_view_pose(eye_relation, view_index, out_pose); +} + +static void +qwerty_destroy(struct xrt_device *xd) +{ + // Note: do not destroy a single device of a qwerty system or its var tracking + // ui will make a null reference + struct qwerty_device *qd = qwerty_device(xd); + qwerty_system_remove(qd->sys, qd); + u_device_free(xd); +} + +struct qwerty_hmd * +qwerty_hmd_create(void) +{ + enum u_device_alloc_flags flags = U_DEVICE_ALLOC_HMD | U_DEVICE_ALLOC_TRACKING_NONE; + size_t num_inputs = 1, num_outputs = 0; + struct qwerty_hmd *qh = U_DEVICE_ALLOCATE(struct qwerty_hmd, flags, num_inputs, num_outputs); + assert(qh); + + struct qwerty_device *qd = &qh->base; + qd->pose.orientation.w = 1.f; + qd->pose.position = QWERTY_HMD_INITIAL_POS; + qd->movement_speed = QWERTY_HMD_INITIAL_MOVEMENT_SPEED; + qd->look_speed = QWERTY_HMD_INITIAL_LOOK_SPEED; + + struct xrt_device *xd = &qd->base; + xd->name = XRT_DEVICE_GENERIC_HMD; + xd->device_type = XRT_DEVICE_TYPE_HMD; + + snprintf(xd->str, XRT_DEVICE_NAME_LEN, QWERTY_HMD_STR); + snprintf(xd->serial, XRT_DEVICE_NAME_LEN, QWERTY_HMD_STR); + + // Fill in xd->hmd + struct u_device_simple_info info; + info.display.w_pixels = 1280; + info.display.h_pixels = 720; + info.display.w_meters = 0.13f; + info.display.h_meters = 0.07f; + info.lens_horizontal_separation_meters = 0.13f / 2.0f; + info.lens_vertical_position_meters = 0.07f / 2.0f; + info.views[0].fov = 85.0f * (M_PI / 180.0f); + info.views[1].fov = 85.0f * (M_PI / 180.0f); + + if (!u_device_setup_split_side_by_side(xd, &info)) { + QWERTY_ERROR(qd, "Failed to setup HMD properties"); + qwerty_destroy(xd); + assert(false); + return NULL; + } + + xd->tracking_origin->type = XRT_TRACKING_TYPE_OTHER; + snprintf(xd->tracking_origin->name, XRT_TRACKING_NAME_LEN, QWERTY_HMD_TRACKER_STR); + + xd->inputs[0].name = XRT_INPUT_GENERIC_HEAD_POSE; + + xd->update_inputs = qwerty_update_inputs; + xd->get_tracked_pose = qwerty_get_tracked_pose; + xd->get_view_pose = qwerty_get_view_pose; + xd->destroy = qwerty_destroy; + u_distortion_mesh_set_none(xd); // Fill in xd->compute_distortion() + + return qh; +} + +struct qwerty_controller * +qwerty_controller_create(bool is_left, struct qwerty_hmd *qhmd) +{ + struct qwerty_controller *qc = U_DEVICE_ALLOCATE(struct qwerty_controller, U_DEVICE_ALLOC_TRACKING_NONE, 4, 1); + assert(qc); + qc->select_clicked = false; + qc->menu_clicked = false; + qc->follow_hmd = qhmd != NULL; + + struct qwerty_device *qd = &qc->base; + qd->pose.orientation.w = 1.f; + qd->pose.position = QWERTY_CONTROLLER_INITIAL_POS(is_left); + qd->movement_speed = QWERTY_CONTROLLER_INITIAL_MOVEMENT_SPEED; + qd->look_speed = QWERTY_CONTROLLER_INITIAL_LOOK_SPEED; + + struct xrt_device *xd = &qd->base; + + xd->name = XRT_DEVICE_SIMPLE_CONTROLLER; + xd->device_type = is_left ? XRT_DEVICE_TYPE_LEFT_HAND_CONTROLLER : XRT_DEVICE_TYPE_RIGHT_HAND_CONTROLLER; + + char *controller_name = is_left ? QWERTY_LEFT_STR : QWERTY_RIGHT_STR; + snprintf(xd->str, XRT_DEVICE_NAME_LEN, "%s", controller_name); + snprintf(xd->serial, XRT_DEVICE_NAME_LEN, "%s", controller_name); + + xd->tracking_origin->type = XRT_TRACKING_TYPE_OTHER; + char *tracker_name = is_left ? QWERTY_LEFT_TRACKER_STR : QWERTY_RIGHT_TRACKER_STR; + snprintf(xd->tracking_origin->name, XRT_TRACKING_NAME_LEN, "%s", tracker_name); + + xd->inputs[QWERTY_SELECT].name = XRT_INPUT_SIMPLE_SELECT_CLICK; + xd->inputs[QWERTY_MENU].name = XRT_INPUT_SIMPLE_MENU_CLICK; + xd->inputs[QWERTY_GRIP].name = XRT_INPUT_SIMPLE_GRIP_POSE; + xd->inputs[QWERTY_AIM].name = XRT_INPUT_SIMPLE_AIM_POSE; // @todo: aim input not implemented + xd->outputs[QWERTY_VIBRATION].name = XRT_OUTPUT_NAME_SIMPLE_VIBRATION; + + xd->update_inputs = qwerty_update_inputs; + xd->get_tracked_pose = qwerty_get_tracked_pose; + xd->set_output = qwerty_set_output; + xd->destroy = qwerty_destroy; + + return qc; +} + +// System methods + +static void +qwerty_setup_var_tracking(struct qwerty_system *qs) +{ + struct qwerty_device *qd_hmd = qs->hmd ? &qs->hmd->base : NULL; + struct qwerty_device *qd_left = &qs->lctrl->base; + struct qwerty_device *qd_right = &qs->rctrl->base; + + u_var_add_root(qs, "Qwerty System", true); + u_var_add_log_level(qs, &qs->ll, "log_level"); + u_var_add_bool(qs, &qs->process_keys, "process_keys"); + + u_var_add_ro_text(qs, "", "Focused Device"); + if (qd_hmd) { + u_var_add_bool(qs, &qs->hmd_focused, "HMD Focused"); + } + u_var_add_bool(qs, &qs->lctrl_focused, "Left Controller Focused"); + u_var_add_bool(qs, &qs->rctrl_focused, "Right Controller Focused"); + + if (qd_hmd) { + u_var_add_gui_header(qs, NULL, qd_hmd->base.str); + u_var_add_pose(qs, &qd_hmd->pose, "hmd.pose"); + u_var_add_f32(qs, &qd_hmd->movement_speed, "hmd.movement_speed"); + u_var_add_f32(qs, &qd_hmd->look_speed, "hmd.look_speed"); + } + + u_var_add_gui_header(qs, NULL, qd_left->base.str); + u_var_add_pose(qs, &qd_left->pose, "left.pose"); + u_var_add_f32(qs, &qd_left->movement_speed, "left.movement_speed"); + u_var_add_f32(qs, &qd_left->look_speed, "left.look_speed"); + + u_var_add_gui_header(qs, NULL, qd_right->base.str); + u_var_add_pose(qs, &qd_right->pose, "right.pose"); + u_var_add_f32(qs, &qd_right->movement_speed, "right.movement_speed"); + u_var_add_f32(qs, &qd_right->look_speed, "right.look_speed"); + + u_var_add_gui_header(qs, NULL, "Help"); + u_var_add_ro_text(qs, "FD: focused device. FC: focused controller.", "Notation"); + u_var_add_ro_text(qs, "HMD is FD by default. Right is FC by default", "Defaults"); + u_var_add_ro_text(qs, "Hold left/right FD", "LCTRL/LALT"); + u_var_add_ro_text(qs, "Move FD", "WASDQE"); + u_var_add_ro_text(qs, "Rotate FD", "Arrow keys"); + u_var_add_ro_text(qs, "Rotate FD", "Hold right click"); + u_var_add_ro_text(qs, "Hold for movement speed", "LSHIFT"); + u_var_add_ro_text(qs, "Modify FD movement speed", "Mouse wheel"); + u_var_add_ro_text(qs, "Modify FD movement speed", "Numpad +/-"); + u_var_add_ro_text(qs, "Reset both or FC pose", "R"); + u_var_add_ro_text(qs, "Toggle both or FC parenting to HMD", "F"); + u_var_add_ro_text(qs, "FC Select click", "Left Click"); + u_var_add_ro_text(qs, "FC Menu click", "Middle Click"); +} + +struct qwerty_system * +qwerty_system_create(struct qwerty_hmd *qhmd, + struct qwerty_controller *qleft, + struct qwerty_controller *qright, + enum u_logging_level log_level) +{ + assert(qleft && "Cannot create a qwerty system when Left controller is NULL"); + assert(qright && "Cannot create a qwerty system when Right controller is NULL"); + + struct qwerty_system *qs = U_TYPED_CALLOC(struct qwerty_system); + qs->hmd = qhmd; + qs->lctrl = qleft; + qs->rctrl = qright; + qs->ll = log_level; + qs->process_keys = true; + + if (qhmd) { + qhmd->base.sys = qs; + } + qleft->base.sys = qs; + qright->base.sys = qs; + + qwerty_setup_var_tracking(qs); + + return qs; +} + +static void +qwerty_system_remove(struct qwerty_system *qs, struct qwerty_device *qd) +{ + if (eq(qd, qs->hmd)) { + qs->hmd = NULL; + } else if (eq(qd, qs->lctrl)) { + qs->lctrl = NULL; + } else if (eq(qd, qs->rctrl)) { + qs->rctrl = NULL; + } else { + assert(false && "Trying to remove a device that is not in the qwerty system"); + } + + bool all_devices_clean = !qs->hmd && !qs->lctrl && !qs->rctrl; + if (all_devices_clean) { + qwerty_system_destroy(qs); + } +} + +static void +qwerty_system_destroy(struct qwerty_system *qs) +{ + bool all_devices_clean = !qs->hmd && !qs->lctrl && !qs->rctrl; + assert(all_devices_clean && "Tried to destroy a qwerty_system without destroying its devices before."); + if (!all_devices_clean) { + return; + } + u_var_remove_root(qs); + free(qs); +} + +// Device methods + +// clang-format off +void qwerty_press_left(struct qwerty_device *qd) { qd->left_pressed = true; } +void qwerty_release_left(struct qwerty_device *qd) { qd->left_pressed = false; } +void qwerty_press_right(struct qwerty_device *qd) { qd->right_pressed = true; } +void qwerty_release_right(struct qwerty_device *qd) { qd->right_pressed = false; } +void qwerty_press_forward(struct qwerty_device *qd) { qd->forward_pressed = true; } +void qwerty_release_forward(struct qwerty_device *qd) { qd->forward_pressed = false; } +void qwerty_press_backward(struct qwerty_device *qd) { qd->backward_pressed = true; } +void qwerty_release_backward(struct qwerty_device *qd) { qd->backward_pressed = false; } +void qwerty_press_up(struct qwerty_device *qd) { qd->up_pressed = true; } +void qwerty_release_up(struct qwerty_device *qd) { qd->up_pressed = false; } +void qwerty_press_down(struct qwerty_device *qd) { qd->down_pressed = true; } +void qwerty_release_down(struct qwerty_device *qd) { qd->down_pressed = false; } + +void qwerty_press_look_left(struct qwerty_device *qd) { qd->look_left_pressed = true; } +void qwerty_release_look_left(struct qwerty_device *qd) { qd->look_left_pressed = false; } +void qwerty_press_look_right(struct qwerty_device *qd) { qd->look_right_pressed = true; } +void qwerty_release_look_right(struct qwerty_device *qd) { qd->look_right_pressed = false; } +void qwerty_press_look_up(struct qwerty_device *qd) { qd->look_up_pressed = true; } +void qwerty_release_look_up(struct qwerty_device *qd) { qd->look_up_pressed = false; } +void qwerty_press_look_down(struct qwerty_device *qd) { qd->look_down_pressed = true; } +void qwerty_release_look_down(struct qwerty_device *qd) { qd->look_down_pressed = false; } +// clang-format on + +void +qwerty_press_sprint(struct qwerty_device *qd) +{ + qd->sprint_pressed = true; +} +void +qwerty_release_sprint(struct qwerty_device *qd) +{ + qd->sprint_pressed = false; +} + +void +qwerty_add_look_delta(struct qwerty_device *qd, float yaw, float pitch) +{ + qd->yaw_delta += yaw * qd->look_speed; + qd->pitch_delta += pitch * qd->look_speed; +} + +void +qwerty_change_movement_speed(struct qwerty_device *qd, float steps) +{ + qd->movement_speed *= powf(MOVEMENT_SPEED_STEP, steps); +} + +void +qwerty_release_all(struct qwerty_device *qd) +{ + qd->left_pressed = false; + qd->right_pressed = false; + qd->forward_pressed = false; + qd->backward_pressed = false; + qd->up_pressed = false; + qd->down_pressed = false; + qd->look_left_pressed = false; + qd->look_right_pressed = false; + qd->look_up_pressed = false; + qd->look_down_pressed = false; + qd->sprint_pressed = false; + qd->yaw_delta = 0; + qd->pitch_delta = 0; +} + +// Controller methods + +// clang-format off +void qwerty_select_click(struct qwerty_controller *qc) { qc->select_clicked = true; } +void qwerty_menu_click(struct qwerty_controller *qc) { qc->menu_clicked = true; } +// clang-format on + +void +qwerty_follow_hmd(struct qwerty_controller *qc, bool follow) +{ + struct qwerty_device *qd = &qc->base; + bool no_qhmd = !qd->sys->hmd; + bool unchanged = qc->follow_hmd == follow; + if (no_qhmd || unchanged) { + return; + } + + struct qwerty_device *qd_hmd = &qd->sys->hmd->base; + struct xrt_space_graph graph = {0}; + struct xrt_space_relation rel = {0}; + + m_space_graph_add_pose(&graph, &qd->pose); + if (follow) { // From global to hmd + m_space_graph_add_inverted_pose_if_not_identity(&graph, &qd_hmd->pose); + } else { // From hmd to global + m_space_graph_add_pose(&graph, &qd_hmd->pose); + } + m_space_graph_resolve(&graph, &rel); + + qd->pose = rel.pose; + qc->follow_hmd = follow; +} + +void +qwerty_reset_controller_pose(struct qwerty_controller *qc) +{ + struct qwerty_device *qd = &qc->base; + + bool no_qhmd = !qd->sys->hmd; + if (no_qhmd) { + return; + } + + bool is_left = qc == qd->sys->lctrl; + + qwerty_follow_hmd(qc, true); + struct xrt_pose pose = {XRT_QUAT_IDENTITY, QWERTY_CONTROLLER_INITIAL_POS(is_left)}; + qd->pose = pose; +} diff --git a/src/xrt/drivers/qwerty/qwerty_device.h b/src/xrt/drivers/qwerty/qwerty_device.h new file mode 100644 index 000000000..96f289a57 --- /dev/null +++ b/src/xrt/drivers/qwerty/qwerty_device.h @@ -0,0 +1,291 @@ +// Copyright 2021, Mateo de Mayo. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Internal header for qwerty_device and its friends. + * @author Mateo de Mayo + * @ingroup drv_qwerty + */ +#pragma once + +#include "util/u_logging.h" +#include "xrt/xrt_device.h" + +#define QWERTY_HMD_STR "Qwerty HMD" +#define QWERTY_HMD_TRACKER_STR QWERTY_HMD_STR " Tracker" +#define QWERTY_LEFT_STR "Qwerty Left Controller" +#define QWERTY_LEFT_TRACKER_STR QWERTY_LEFT_STR " Tracker" +#define QWERTY_RIGHT_STR "Qwerty Right Controller" +#define QWERTY_RIGHT_TRACKER_STR QWERTY_RIGHT_STR " Tracker" + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * @addtogroup drv_qwerty + * @{ + */ + +/*! + * @brief Container of qwerty devices and driver properties. + * @see qwerty_hmd, qwerty_controller + */ +struct qwerty_system +{ + struct qwerty_hmd *hmd; //!< Can be NULL + struct qwerty_controller *lctrl; //!< Cannot be NULL + struct qwerty_controller *rctrl; //!< Cannot be NULL + enum u_logging_level ll; + bool process_keys; //!< If false disable keyboard and mouse input + bool hmd_focused; //!< For gui var tracking only, true if hmd is the focused device + bool lctrl_focused; //!< Same as `hmd_focused` but for the left controller + bool rctrl_focused; //!< Same as `hmd_focused` but for the right controller +}; + +/*! + * Fake device that modifies its tracked pose through its methods. + * @implements xrt_device + */ +struct qwerty_device +{ + struct xrt_device base; + struct xrt_pose pose; //!< Internal pose state + struct qwerty_system *sys; //!< Reference to the system this device is in. + + float movement_speed; //!< In meters per frame + bool left_pressed; + bool right_pressed; + bool forward_pressed; + bool backward_pressed; + bool up_pressed; + bool down_pressed; + + float look_speed; //!< In radians per frame + bool look_left_pressed; + bool look_right_pressed; + bool look_up_pressed; + bool look_down_pressed; + + bool sprint_pressed; //!< Movement speed boost + float yaw_delta; //!< How much extra yaw to add for the next pose. Then reset to 0. + float pitch_delta; //!< Similar to `yaw_delta` +}; + +/*! + * @implements qwerty_device + * @see qwerty_system + */ +struct qwerty_hmd +{ + struct qwerty_device base; +}; + +/*! + * Supports input actions and can be attached to the HMD pose. + * @implements qwerty_device + * @see qwerty_system + */ +struct qwerty_controller +{ + struct qwerty_device base; + + bool select_clicked; + bool menu_clicked; + + /*! + * Only used when a qwerty_hmd exists in the system. + * Do not modify directly; use qwerty_follow_hmd(). + * If true, `pose` is relative to the qwerty_hmd. + */ + bool follow_hmd; // @todo: Make this work with non-qwerty HMDs. +}; + +/*! + * @public @memberof qwerty_system + */ +struct qwerty_system * +qwerty_system_create(struct qwerty_hmd *qhmd, + struct qwerty_controller *qleft, + struct qwerty_controller *qright, + enum u_logging_level log_level); + +/* + * + * qwerty_device methods + * + */ + +/*! + * @brief Cast to qwerty_device. Ensures returning a valid device or crashing. + * @public @memberof qwerty_device + */ +struct qwerty_device * +qwerty_device(struct xrt_device *xd); + +//! @public @memberof qwerty_device +void +qwerty_press_left(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_release_left(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_press_right(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_release_right(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_press_forward(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_release_forward(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_press_backward(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_release_backward(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_press_up(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_release_up(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_press_down(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_release_down(struct qwerty_device *qd); + +//! @public @memberof qwerty_device +void +qwerty_press_look_left(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_release_look_left(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_press_look_right(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_release_look_right(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_press_look_up(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_release_look_up(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_press_look_down(struct qwerty_device *qd); +//! @public @memberof qwerty_device +void +qwerty_release_look_down(struct qwerty_device *qd); + +/*! + * Momentarily increase `movement_speed` until `qwerty_release_sprint()` + * @public @memberof qwerty_device + */ +void +qwerty_press_sprint(struct qwerty_device *qd); + +/*! + * Stop doing what @ref qwerty_press_sprint started. + * @public @memberof qwerty_device + */ +void +qwerty_release_sprint(struct qwerty_device *qd); + +/*! + * Add yaw and pitch movement for the next frame + * @public @memberof qwerty_device + */ +void +qwerty_add_look_delta(struct qwerty_device *qd, float yaw, float pitch); + +/*! + * Change movement speed in exponential steps (usually integers, but any float allowed) + * @public @memberof qwerty_device + */ +void +qwerty_change_movement_speed(struct qwerty_device *qd, float steps); + +/*! + * Release all movement input + * @public @memberof qwerty_device + */ +void +qwerty_release_all(struct qwerty_device *qd); + +/*! + * Create qwerty_hmd. Crash on failure. + * @public @memberof qwerty_hmd + */ +struct qwerty_hmd * +qwerty_hmd_create(void); + +/*! + * Cast to qwerty_hmd. Ensures returning a valid HMD or crashing. + * @public @memberof qwerty_hmd + */ +struct qwerty_hmd * +qwerty_hmd(struct xrt_device *xd); + +/* + * + * qwerty_controller methods + * + */ +/*! + * Create qwerty_controller. Crash on failure. + * @public @memberof qwerty_controller + */ +struct qwerty_controller * +qwerty_controller_create(bool is_left, struct qwerty_hmd *qhmd); + +/*! + * Cast to qwerty_controller. Ensures returning a valid controller or crashing. + * @public @memberof qwerty_controller + */ +struct qwerty_controller * +qwerty_controller(struct xrt_device *xd); + +/*! + * Simulate input/select/click + * @public @memberof qwerty_controller + */ +void +qwerty_select_click(struct qwerty_controller *qc); + +/*! + * Simulate input/menu/click + * @public @memberof qwerty_controller + */ +void +qwerty_menu_click(struct qwerty_controller *qc); + +/*! + * Attach/detach the pose of `qc` to its HMD. Only works when a qwerty_hmd is present. + * @public @memberof qwerty_controller + */ +void +qwerty_follow_hmd(struct qwerty_controller *qc, bool follow); + +/*! + * Reset controller to initial pose and makes it follow the HMD + * @public @memberof qwerty_controller + */ +void +qwerty_reset_controller_pose(struct qwerty_controller *qc); + + +/*! + * @} + */ + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/qwerty/qwerty_interface.h b/src/xrt/drivers/qwerty/qwerty_interface.h new file mode 100644 index 000000000..eff6cd732 --- /dev/null +++ b/src/xrt/drivers/qwerty/qwerty_interface.h @@ -0,0 +1,54 @@ +// Copyright 2021, Mateo de Mayo. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Interface to @ref drv_qwerty. + * @author Mateo de Mayo + * @ingroup drv_qwerty + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +typedef union SDL_Event SDL_Event; + +/*! + * @defgroup drv_qwerty Qwerty driver + * @ingroup drv + * + * @brief Driver for emulated HMD and controllers through keyboard and mouse. + * @{ + */ + +//! Create an auto prober for qwerty devices. +struct xrt_auto_prober * +qwerty_create_auto_prober(void); + +/*! + * Process an SDL_Event (like a key press) and dispatches a suitable action + * to the appropriate qwerty_device. + * + * @note A qwerty_controller might not be in use (for example if you have + * physical controllers connected), though its memory will be modified by these + * events regardless. A qwerty_hmd not in use will not be modified as it never + * gets created. + */ +void +qwerty_process_event(struct xrt_device **xdevs, size_t num_xdevs, SDL_Event event); + +/*! + * @} + */ + +/*! + * @dir drivers/qwerty + * + * @brief @ref drv_qwerty files. + */ + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/qwerty/qwerty_prober.c b/src/xrt/drivers/qwerty/qwerty_prober.c new file mode 100644 index 000000000..35f26f3dc --- /dev/null +++ b/src/xrt/drivers/qwerty/qwerty_prober.c @@ -0,0 +1,84 @@ +// Copyright 2021, Mateo de Mayo. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Qwerty devices @ref xrt_auto_prober "autoprober". + * @author Mateo de Mayo + * @ingroup drv_qwerty + */ + +#include "qwerty_device.h" +#include "util/u_misc.h" +#include "util/u_debug.h" +#include "util/u_logging.h" +#include "xrt/xrt_prober.h" + +// Using INFO as default to inform events real devices could report physically +DEBUG_GET_ONCE_LOG_OPTION(qwerty_log, "QWERTY_LOG", U_LOGGING_INFO) + +// Driver disabled by default for being experimental +DEBUG_GET_ONCE_BOOL_OPTION(qwerty_enable, "QWERTY_ENABLE", false) + +struct qwerty_prober +{ + struct xrt_auto_prober base; +}; + +static struct qwerty_prober * +qwerty_prober(struct xrt_auto_prober *p) +{ + return (struct qwerty_prober *)p; +} + +static void +qwerty_prober_destroy(struct xrt_auto_prober *p) +{ + struct qwerty_prober *qp = qwerty_prober(p); + free(qp); +} + +static int +qwerty_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) +{ + bool qwerty_enabled = debug_get_bool_option_qwerty_enable(); + if (!qwerty_enabled) { + return 0; + } + + bool hmd_wanted = !no_hmds; // Hopefully easier to reason about + + struct qwerty_hmd *qhmd = hmd_wanted ? qwerty_hmd_create() : NULL; + struct qwerty_controller *qleft = qwerty_controller_create(true, qhmd); + struct qwerty_controller *qright = qwerty_controller_create(false, qhmd); + + enum u_logging_level log_level = debug_get_log_option_qwerty_log(); + qwerty_system_create(qhmd, qleft, qright, log_level); + + struct xrt_device *xd_hmd = &qhmd->base.base; + struct xrt_device *xd_left = &qleft->base.base; + struct xrt_device *xd_right = &qright->base.base; + + if (hmd_wanted) { + out_xdevs[0] = xd_hmd; + } + out_xdevs[1 - !hmd_wanted] = xd_left; + out_xdevs[2 - !hmd_wanted] = xd_right; + + int num_qwerty_devices = hmd_wanted + 2; + return num_qwerty_devices; +} + +struct xrt_auto_prober * +qwerty_create_auto_prober() +{ + struct qwerty_prober *qp = U_TYPED_CALLOC(struct qwerty_prober); + qp->base.name = "Qwerty"; + qp->base.destroy = qwerty_prober_destroy; + qp->base.lelo_dallas_autoprobe = qwerty_prober_autoprobe; + + return &qp->base; +} diff --git a/src/xrt/drivers/qwerty/qwerty_sdl.c b/src/xrt/drivers/qwerty/qwerty_sdl.c new file mode 100644 index 000000000..80e5c7cff --- /dev/null +++ b/src/xrt/drivers/qwerty/qwerty_sdl.c @@ -0,0 +1,234 @@ +// Copyright 2021, Mateo de Mayo. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Connection between user-generated SDL events and qwerty devices. + * @author Mateo de Mayo + * @ingroup drv_qwerty + */ + +#include "qwerty_device.h" +#include "util/u_device.h" +#include "xrt/xrt_device.h" +#include +#include + +// Amount of look_speed units a mouse delta of 1px in screen space will rotate the device +#define SENSITIVITY 0.1f + +static struct qwerty_system * +find_qwerty_system(struct xrt_device **xdevs, size_t num_xdevs) +{ + struct xrt_device *xdev = NULL; + for (size_t i = 0; i < num_xdevs; i++) { + if (xdevs[i] == NULL) { + continue; + } + // We check against tracker name instead of device name because the tracking overrides + // cause the multi device to have the same names even though they are not qwerty devices. + const char *tracker_name = xdevs[i]->tracking_origin->name; + if (strcmp(tracker_name, QWERTY_HMD_TRACKER_STR) == 0 || + strcmp(tracker_name, QWERTY_LEFT_TRACKER_STR) == 0 || + strcmp(tracker_name, QWERTY_RIGHT_TRACKER_STR) == 0) { + xdev = xdevs[i]; + break; + } + } + + assert(xdev != NULL && "There is no device in xdevs with the name of a qwerty device"); + struct qwerty_device *qdev = qwerty_device(xdev); + struct qwerty_system *qsys = qdev->sys; + assert(qsys != NULL && "The qwerty_system of a qwerty_device was null"); + return qsys; +} + +// Determines the default qwerty device based on which devices are in use +static struct qwerty_device * +default_qwerty_device(struct xrt_device **xdevs, size_t num_xdevs, struct qwerty_system *qsys) +{ + int head, left, right; + head = left = right = XRT_DEVICE_ROLE_UNASSIGNED; + u_device_assign_xdev_roles(xdevs, num_xdevs, &head, &left, &right); + + struct xrt_device *xd_hmd = qsys->hmd ? &qsys->hmd->base.base : NULL; + struct xrt_device *xd_left = &qsys->lctrl->base.base; + struct xrt_device *xd_right = &qsys->rctrl->base.base; + + struct qwerty_device *default_qdev = NULL; + if (xdevs[head] == xd_hmd) { + default_qdev = qwerty_device(xd_hmd); + } else if (xdevs[right] == xd_right) { + default_qdev = qwerty_device(xd_right); + } else if (xdevs[left] == xd_left) { + default_qdev = qwerty_device(xd_left); + } else { // Even here, xd_right is allocated and so we can modify it + default_qdev = qwerty_device(xd_right); + } + + return default_qdev; +} + +// Determines the default qwerty controller based on which devices are in use +static struct qwerty_controller * +default_qwerty_controller(struct xrt_device **xdevs, size_t num_xdevs, struct qwerty_system *qsys) +{ + int head, left, right; + head = left = right = XRT_DEVICE_ROLE_UNASSIGNED; + u_device_assign_xdev_roles(xdevs, num_xdevs, &head, &left, &right); + + struct xrt_device *xd_left = &qsys->lctrl->base.base; + struct xrt_device *xd_right = &qsys->rctrl->base.base; + + struct qwerty_controller *default_qctrl = NULL; + if (xdevs[right] == xd_right) { + default_qctrl = qwerty_controller(xd_right); + } else if (xdevs[left] == xd_left) { + default_qctrl = qwerty_controller(xd_left); + } else { // Even here, xd_right is allocated and so we can modify it + default_qctrl = qwerty_controller(xd_right); + } + + return default_qctrl; +} + +void +qwerty_process_event(struct xrt_device **xdevs, size_t num_xdevs, SDL_Event event) +{ + static struct qwerty_system *qsys = NULL; + + static bool alt_pressed = false; + static bool ctrl_pressed = false; + + // Default focused device: the one focused when CTRL and ALT are not pressed + static struct qwerty_device *default_qdev; + // Default focused controller: the one used for qwerty_controller specific methods + static struct qwerty_controller *default_qctrl; + + // We can cache the devices as they don't get destroyed during runtime + static bool cached = false; + if (!cached) { + qsys = find_qwerty_system(xdevs, num_xdevs); + default_qdev = default_qwerty_device(xdevs, num_xdevs, qsys); + default_qctrl = default_qwerty_controller(xdevs, num_xdevs, qsys); + cached = true; + } + + if (!qsys->process_keys) { + return; + } + + // Initialize different views of the same pointers. + + struct qwerty_controller *qleft = qsys->lctrl; + struct qwerty_device *qd_left = &qleft->base; + + struct qwerty_controller *qright = qsys->rctrl; + struct qwerty_device *qd_right = &qright->base; + + bool using_qhmd = qsys->hmd != NULL; + struct qwerty_hmd *qhmd = using_qhmd ? qsys->hmd : NULL; + struct qwerty_device *qd_hmd = using_qhmd ? &qhmd->base : NULL; + + // clang-format off + // CTRL/ALT keys logic + bool alt_down = event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_LALT; + bool alt_up = event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_LALT; + bool ctrl_down = event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_LCTRL; + bool ctrl_up = event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_LCTRL; + if (alt_down) alt_pressed = true; + if (alt_up) alt_pressed = false; + if (ctrl_down) ctrl_pressed = true; + if (ctrl_up) ctrl_pressed = false; + + bool change_focus = alt_down || alt_up || ctrl_down || ctrl_up; + if (change_focus) { + if (using_qhmd) qwerty_release_all(qd_hmd); + qwerty_release_all(qd_right); + qwerty_release_all(qd_left); + } + + // Determine focused device + struct qwerty_device *qdev; + if (ctrl_pressed) qdev = qd_left; + else if (alt_pressed) qdev = qd_right; + else qdev = default_qdev; + + // Determine focused controller for qwerty_controller specific methods + struct qwerty_controller *qctrl = qdev != qd_hmd ? qwerty_controller(&qdev->base) : default_qctrl; + + // Update gui tracked variables + qsys->hmd_focused = qdev == qd_hmd; + qsys->lctrl_focused = qdev == qd_left; + qsys->rctrl_focused = qdev == qd_right; + + // WASDQE Movement + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_a) qwerty_press_left(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_a) qwerty_release_left(qdev); + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_d) qwerty_press_right(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_d) qwerty_release_right(qdev); + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_w) qwerty_press_forward(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_w) qwerty_release_forward(qdev); + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_s) qwerty_press_backward(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_s) qwerty_release_backward(qdev); + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_e) qwerty_press_up(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_e) qwerty_release_up(qdev); + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_q) qwerty_press_down(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_q) qwerty_release_down(qdev); + + // Arrow keys rotation + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_LEFT) qwerty_press_look_left(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_LEFT) qwerty_release_look_left(qdev); + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_RIGHT) qwerty_press_look_right(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_RIGHT) qwerty_release_look_right(qdev); + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_UP) qwerty_press_look_up(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_UP) qwerty_release_look_up(qdev); + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_DOWN) qwerty_press_look_down(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_DOWN) qwerty_release_look_down(qdev); + + // Movement speed + if (event.type == SDL_MOUSEWHEEL) qwerty_change_movement_speed(qdev, event.wheel.y); + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_KP_PLUS) qwerty_change_movement_speed(qdev, 1); + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_KP_MINUS) qwerty_change_movement_speed(qdev, -1); + + // Sprinting + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_LSHIFT) qwerty_press_sprint(qdev); + if (event.type == SDL_KEYUP && event.key.keysym.sym == SDLK_LSHIFT) qwerty_release_sprint(qdev); + + // Mouse rotation + if (event.type == SDL_MOUSEBUTTONUP && event.button.button == SDL_BUTTON_RIGHT) { + SDL_SetRelativeMouseMode(false); + } + if (event.type == SDL_MOUSEMOTION && event.motion.state & SDL_BUTTON_RMASK) { + SDL_SetRelativeMouseMode(true); + float yaw = -event.motion.xrel * SENSITIVITY; + float pitch = -event.motion.yrel * SENSITIVITY; + qwerty_add_look_delta(qdev, yaw, pitch); + } + + // Select and menu clicks only for controllers. + if (event.type == SDL_MOUSEBUTTONDOWN && event.button.button == SDL_BUTTON_LEFT) qwerty_select_click(qctrl); + if (event.type == SDL_MOUSEBUTTONDOWN && event.button.button == SDL_BUTTON_MIDDLE) qwerty_menu_click(qctrl); + + // clang-format on + + // Controllers follow/unfollow HMD + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_f && event.key.repeat == 0) { + if (qdev != qd_hmd) { + qwerty_follow_hmd(qctrl, !qctrl->follow_hmd); + } else { // If no controller is focused, set both to the same state + bool both_not_following = !qleft->follow_hmd && !qright->follow_hmd; + qwerty_follow_hmd(qleft, both_not_following); + qwerty_follow_hmd(qright, both_not_following); + } + } + + // Reset controller poses + if (event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_r && event.key.repeat == 0) { + if (qdev != qd_hmd) { + qwerty_reset_controller_pose(qctrl); + } else { // If no controller is focused, reset both + qwerty_reset_controller_pose(qleft); + qwerty_reset_controller_pose(qright); + } + } +} diff --git a/src/xrt/drivers/realsense/rs_6dof.c b/src/xrt/drivers/realsense/rs_6dof.c deleted file mode 100644 index 67d5265fe..000000000 --- a/src/xrt/drivers/realsense/rs_6dof.c +++ /dev/null @@ -1,418 +0,0 @@ -// Copyright 2020, Collabora, Ltd. -// Copyright 2020, Nova King. -// SPDX-License-Identifier: BSL-1.0 -/*! - * @file - * @brief RealSense helper driver for 6DOF tracking. - * @author Nova King - * @author Jakob Bornecrantz - * @ingroup drv_rs - */ - -#include "xrt/xrt_defines.h" -#include "xrt/xrt_device.h" -#include "math/m_api.h" -#include "math/m_space.h" -#include "math/m_predict.h" - -#include "os/os_time.h" -#include "os/os_threading.h" - -#include "util/u_time.h" -#include "util/u_device.h" -#include "util/u_logging.h" - -#include -#include -#include -#include - -#include -#include -#include -#include - - -/*! - * Stupid convenience macro to print out a pose, only used for debugging - */ -#define print_pose(msg, pose) \ - U_LOG_E(msg " %f %f %f %f %f %f %f", pose.position.x, pose.position.y, pose.position.z, pose.orientation.x, \ - pose.orientation.y, pose.orientation.z, pose.orientation.w) - -/*! - * @implements xrt_device - */ -struct rs_6dof -{ - struct xrt_device base; - - uint64_t relation_timestamp_ns; - struct xrt_space_relation relation; - - // arbitrary offset to apply to the pose the t265 gives us - struct xrt_pose offset; - - struct os_thread_helper oth; - - rs2_context *ctx; - rs2_pipeline *pipe; - rs2_pipeline_profile *profile; - rs2_config *config; -}; - - -/*! - * Helper to convert a xdev to a @ref rs_6dof. - */ -static inline struct rs_6dof * -rs_6dof(struct xrt_device *xdev) -{ - return (struct rs_6dof *)xdev; -} - -/*! - * Simple helper to check and print error messages. - */ -static int -check_error(struct rs_6dof *rs, rs2_error *e) -{ - if (e == NULL) { - return 0; - } - - U_LOG_E("rs_error was raised when calling %s(%s):", rs2_get_failed_function(e), rs2_get_failed_args(e)); - U_LOG_E("%s", rs2_get_error_message(e)); - - return 1; -} - -/*! - * Frees all RealSense resources. - */ -static void -close_6dof(struct rs_6dof *rs) -{ - if (rs->config) { - rs2_delete_config(rs->config); - rs->config = NULL; - } - - if (rs->profile) { - rs2_delete_pipeline_profile(rs->profile); - rs->profile = NULL; - } - - if (rs->pipe) { - rs2_pipeline_stop(rs->pipe, NULL); - rs2_delete_pipeline(rs->pipe); - rs->pipe = NULL; - } - - if (rs->ctx) { - rs2_delete_context(rs->ctx); - rs->ctx = NULL; - } -} - -/*! - * Create all RealSense resources needed for 6DOF tracking. - */ -static int -create_6dof(struct rs_6dof *rs) -{ - assert(rs != NULL); - rs2_error *e = NULL; - - rs->ctx = rs2_create_context(RS2_API_VERSION, &e); - if (check_error(rs, e) != 0) { - close_6dof(rs); - return 1; - } - - rs->pipe = rs2_create_pipeline(rs->ctx, &e); - if (check_error(rs, e) != 0) { - close_6dof(rs); - return 1; - } - - rs->config = rs2_create_config(&e); - if (check_error(rs, e) != 0) { - close_6dof(rs); - return 1; - } - - rs2_config_enable_stream(rs->config, // - RS2_STREAM_POSE, // Type - 0, // Index - 0, // Width - 0, // Height - RS2_FORMAT_6DOF, // Format - 200, // FPS - &e); - if (check_error(rs, e) != 0) { - close_6dof(rs); - return 1; - } - - rs->profile = rs2_pipeline_start_with_config(rs->pipe, rs->config, &e); - if (check_error(rs, e) != 0) { - close_6dof(rs); - return 1; - } - - return 0; -} - - -void -rs_update_offset(struct xrt_pose offset, struct xrt_device *xdev) -{ - struct rs_6dof *rs = rs_6dof(xdev); - memcpy(&rs->offset, &offset, sizeof(struct xrt_pose)); -} - - -/*! - * Process a frame as 6DOF data, does not assume ownership of the frame. - */ -static void -process_frame(struct rs_6dof *rs, rs2_frame *frame) -{ - rs2_error *e = NULL; - int ret = 0; - - ret = rs2_is_frame_extendable_to(frame, RS2_EXTENSION_POSE_FRAME, &e); - if (check_error(rs, e) != 0 || ret == 0) { - return; - } - - rs2_pose camera_pose; - rs2_pose_frame_get_pose_data(frame, &camera_pose, &e); - if (check_error(rs, e) != 0) { - return; - } - -#if 0 - rs2_timestamp_domain domain = rs2_get_frame_timestamp_domain(frame, &e); - if (check_error(rs, e) != 0) { - return; - } -#endif - - double timestamp_miliseconds = rs2_get_frame_timestamp(frame, &e); - if (check_error(rs, e) != 0) { - return; - } - - // Close enough - uint64_t now_real_ns = os_realtime_get_ns(); - uint64_t now_monotonic_ns = os_monotonic_get_ns(); - uint64_t timestamp_ns = (uint64_t)(timestamp_miliseconds * 1000.0 * 1000.0); - - // How far in the past is it? - uint64_t diff_ns = now_real_ns - timestamp_ns; - - // Adjust the timestamp to monotonic time. - timestamp_ns = now_monotonic_ns - diff_ns; - - /* - * Transfer the data to the struct. - */ - - // Re-use the thread lock for the data. - os_thread_helper_lock(&rs->oth); - - // clang-format off - // Timestamp - rs->relation_timestamp_ns = timestamp_ns; - - // Rotation/angular - rs->relation.pose.orientation.x = camera_pose.rotation.x; - rs->relation.pose.orientation.y = camera_pose.rotation.y; - rs->relation.pose.orientation.z = camera_pose.rotation.z; - rs->relation.pose.orientation.w = camera_pose.rotation.w; - rs->relation.angular_velocity.x = camera_pose.angular_velocity.x; - rs->relation.angular_velocity.y = camera_pose.angular_velocity.y; - rs->relation.angular_velocity.z = camera_pose.angular_velocity.z; - - // Position/linear - rs->relation.pose.position.x = camera_pose.translation.x; - rs->relation.pose.position.y = camera_pose.translation.y; - rs->relation.pose.position.z = camera_pose.translation.z; - rs->relation.linear_velocity.x = camera_pose.velocity.x; - rs->relation.linear_velocity.y = camera_pose.velocity.y; - rs->relation.linear_velocity.z = camera_pose.velocity.z; - - rs->relation.relation_flags = - XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | - XRT_SPACE_RELATION_POSITION_VALID_BIT | - XRT_SPACE_RELATION_LINEAR_VELOCITY_VALID_BIT | - XRT_SPACE_RELATION_ANGULAR_VELOCITY_VALID_BIT | - XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | - XRT_SPACE_RELATION_POSITION_TRACKED_BIT; - // clang-format on - - // Re-use the thread lock for the data. - os_thread_helper_unlock(&rs->oth); -} - -static int -update(struct rs_6dof *rs) -{ - rs2_frame *frames; - rs2_error *e = NULL; - - frames = rs2_pipeline_wait_for_frames(rs->pipe, RS2_DEFAULT_TIMEOUT, &e); - if (check_error(rs, e) != 0) { - return 1; - } - - int num_frames = rs2_embedded_frames_count(frames, &e); - if (check_error(rs, e) != 0) { - return 1; - } - - for (int i = 0; i < num_frames; i++) { - rs2_frame *frame = rs2_extract_frame(frames, i, &e); - if (check_error(rs, e) != 0) { - rs2_release_frame(frames); - return 1; - } - - // Does not assume ownership of the frame. - process_frame(rs, frame); - rs2_release_frame(frame); - - rs2_release_frame(frames); - } - - return 0; -} - -static void * -rs_run_thread(void *ptr) -{ - struct rs_6dof *rs = (struct rs_6dof *)ptr; - - os_thread_helper_lock(&rs->oth); - - while (os_thread_helper_is_running_locked(&rs->oth)) { - - os_thread_helper_unlock(&rs->oth); - - int ret = update(rs); - if (ret < 0) { - return NULL; - } - } - - return NULL; -} - -static void -rs_6dof_update_inputs(struct xrt_device *xdev) -{ - // Empty -} - -static void -rs_6dof_get_tracked_pose(struct xrt_device *xdev, - enum xrt_input_name name, - uint64_t at_timestamp_ns, - struct xrt_space_relation *out_relation) -{ - struct rs_6dof *rs = rs_6dof(xdev); - - if (name != XRT_INPUT_GENERIC_HEAD_POSE) { - U_LOG_E("unknown input name"); - return; - } - - os_thread_helper_lock(&rs->oth); - struct xrt_space_relation relation_not_predicted = rs->relation; - uint64_t relation_timestamp_ns = rs->relation_timestamp_ns; - os_thread_helper_unlock(&rs->oth); - - int64_t diff_prediction_ns = 0; - diff_prediction_ns = at_timestamp_ns - relation_timestamp_ns; - struct xrt_space_relation relation; - - double delta_s = time_ns_to_s(diff_prediction_ns); - m_predict_relation(&relation_not_predicted, delta_s, &relation); - - struct xrt_space_graph xsg = {0}; - m_space_graph_add_pose(&xsg, &rs->offset); - m_space_graph_add_relation(&xsg, &relation); - m_space_graph_resolve(&xsg, out_relation); -} -static void -rs_6dof_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, - uint32_t view_index, - struct xrt_pose *out_pose) -{ - assert(false); -} - -static void -rs_6dof_destroy(struct xrt_device *xdev) -{ - struct rs_6dof *rs = rs_6dof(xdev); - - // Destroy the thread object. - os_thread_helper_destroy(&rs->oth); - - close_6dof(rs); - - free(rs); -} - -struct xrt_device * -rs_6dof_create(void) -{ - struct rs_6dof *rs = U_DEVICE_ALLOCATE(struct rs_6dof, U_DEVICE_ALLOC_TRACKING_NONE, 1, 0); - int ret = 0; - - rs->base.update_inputs = rs_6dof_update_inputs; - rs->base.get_tracked_pose = rs_6dof_get_tracked_pose; - rs->base.get_view_pose = rs_6dof_get_view_pose; - rs->base.destroy = rs_6dof_destroy; - rs->base.name = XRT_DEVICE_GENERIC_HMD; // This is a lie. - rs->relation.pose.orientation.w = 1.0f; // All other values set to zero. - - rs->base.tracking_origin->type = XRT_TRACKING_TYPE_EXTERNAL_SLAM; - - // Print name. - snprintf(rs->base.str, XRT_DEVICE_NAME_LEN, "Intel RealSense 6-DOF"); - - // Setup input, this is a lie. - rs->base.inputs[0].name = XRT_INPUT_GENERIC_HEAD_POSE; - - // Thread and other state. - ret = os_thread_helper_init(&rs->oth); - if (ret != 0) { - U_LOG_E("Failed to init threading!"); - rs_6dof_destroy(&rs->base); - return NULL; - } - - ret = create_6dof(rs); - if (ret != 0) { - rs_6dof_destroy(&rs->base); - return NULL; - } - - ret = os_thread_helper_start(&rs->oth, rs_run_thread, rs); - if (ret != 0) { - U_LOG_E("Failed to start thread!"); - rs_6dof_destroy(&rs->base); - return NULL; - } - - rs->base.orientation_tracking_supported = true; - rs->base.position_tracking_supported = true; - rs->base.device_type = XRT_DEVICE_TYPE_GENERIC_TRACKER; - - return &rs->base; -} diff --git a/src/xrt/drivers/realsense/rs_ddev.c b/src/xrt/drivers/realsense/rs_ddev.c new file mode 100644 index 000000000..9456586ff --- /dev/null +++ b/src/xrt/drivers/realsense/rs_ddev.c @@ -0,0 +1,507 @@ +// Copyright 2020, Collabora, Ltd. +// Copyright 2020, Nova King. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief RealSense helper driver for in-device SLAM 6DOF tracking. + * @author Moses Turner + * @author Nova King + * @author Jakob Bornecrantz + * @ingroup drv_rs + */ + +#include "xrt/xrt_defines.h" +#include "xrt/xrt_device.h" +#include "math/m_api.h" +#include "math/m_space.h" +#include "math/m_predict.h" +#include "math/m_relation_history.h" + +#include "os/os_time.h" +#include "os/os_threading.h" + +#include "util/u_time.h" +#include "util/u_device.h" +#include "util/u_logging.h" + +#include "util/u_json.h" +#include "util/u_config_json.h" + +#include "rs_driver.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + + +/*! + * Stupid convenience macro to print out a pose, only used for debugging + */ +#define print_pose(msg, pose) \ + U_LOG_E(msg " %f %f %f %f %f %f %f", pose.position.x, pose.position.y, pose.position.z, pose.orientation.x, \ + pose.orientation.y, pose.orientation.z, pose.orientation.w) + +/*! + * Device-SLAM tracked RealSense device (T26X series). + * + * @implements xrt_device + */ +struct rs_ddev +{ + struct xrt_device base; + + struct m_relation_history *relation_hist; + + struct os_thread_helper oth; + + bool enable_mapping; + bool enable_pose_jumping; + bool enable_relocalization; + bool enable_pose_prediction; + bool enable_pose_filtering; //!< Forward compatibility for when that 1-euro filter is working + + struct rs_container rsc; //!< Container of realsense API related objects +}; + + +/*! + * Helper to convert a xdev to a @ref rs_ddev. + */ +static inline struct rs_ddev * +rs_ddev(struct xrt_device *xdev) +{ + return (struct rs_ddev *)xdev; +} + +/*! + * Simple helper to check and print error messages. + */ +static int +check_error(struct rs_ddev *rs, rs2_error *e) +{ + if (e == NULL) { + return 0; + } + + U_LOG_E("rs_error was raised when calling %s(%s):", rs2_get_failed_function(e), rs2_get_failed_args(e)); + U_LOG_E("%s", rs2_get_error_message(e)); + + return 1; +} + +/*! + * Frees all RealSense resources. + */ +static void +close_ddev(struct rs_ddev *rs) +{ + struct rs_container *rsc = &rs->rsc; + rs2_pipeline_stop(rsc->pipeline, NULL); + rs_container_cleanup(&rs->rsc); +} + +#define CHECK_RS2() \ + do { \ + if (check_error(rs, e) != 0) { \ + close_ddev(rs); \ + return 1; \ + } \ + } while (0) + + +/*! + * Create all RealSense resources needed for 6DOF tracking. + */ +static int +create_ddev(struct rs_ddev *rs, int device_idx) +{ + assert(rs != NULL); + rs2_error *e = NULL; + + struct rs_container *rsc = &rs->rsc; + + rsc->context = rs2_create_context(RS2_API_VERSION, &e); + CHECK_RS2(); + + rsc->device_list = rs2_query_devices(rsc->context, &e); + CHECK_RS2(); + + rsc->pipeline = rs2_create_pipeline(rsc->context, &e); + CHECK_RS2(); + + rsc->config = rs2_create_config(&e); + CHECK_RS2(); + + // Set the pipeline to start specifically on the realsense device the prober selected + rsc->device_idx = device_idx; + rsc->device = rs2_create_device(rsc->device_list, rsc->device_idx, &e); + CHECK_RS2(); + + bool ddev_has_serial = rs2_supports_device_info(rsc->device, RS2_CAMERA_INFO_SERIAL_NUMBER, &e); + CHECK_RS2(); + + if (ddev_has_serial) { + + const char *ddev_serial = rs2_get_device_info(rsc->device, RS2_CAMERA_INFO_SERIAL_NUMBER, &e); + CHECK_RS2(); + + rs2_config_enable_device(rsc->config, ddev_serial, &e); + CHECK_RS2(); + + } else { + U_LOG_W("Unexpected, the realsense device in use does not provide a serial number."); + } + + rs2_delete_device(rsc->device); + + rs2_config_enable_stream(rsc->config, // + RS2_STREAM_POSE, // Type + 0, // Index + 0, // Width + 0, // Height + RS2_FORMAT_6DOF, // Format + 200, // FPS + &e); + CHECK_RS2(); + + rsc->profile = rs2_config_resolve(rsc->config, rsc->pipeline, &e); + CHECK_RS2(); + + rsc->device = rs2_pipeline_profile_get_device(rsc->profile, &e); + CHECK_RS2(); + + rs2_sensor_list *sensors = rs2_query_sensors(rsc->device, &e); + CHECK_RS2(); + + //! @todo 0 index hardcoded, check device with RS2_EXTENSION_POSE_SENSOR or similar instead + rs2_sensor *sensor = rs2_create_sensor(sensors, 0, &e); + CHECK_RS2(); + + rs2_set_option((rs2_options *)sensor, RS2_OPTION_ENABLE_MAPPING, rs->enable_mapping ? 1.0f : 0.0f, &e); + CHECK_RS2(); + + if (rs->enable_mapping) { + // Neither of these options mean anything if mapping is off; in fact it + // errors out if we mess with these with mapping off + rs2_set_option((rs2_options *)sensor, RS2_OPTION_ENABLE_RELOCALIZATION, + rs->enable_relocalization ? 1.0f : 0.0f, &e); + CHECK_RS2(); + + rs2_set_option((rs2_options *)sensor, RS2_OPTION_ENABLE_POSE_JUMPING, + rs->enable_pose_jumping ? 1.0f : 0.0f, &e); + CHECK_RS2(); + } + + rsc->profile = rs2_pipeline_start_with_config(rsc->pipeline, rsc->config, &e); + CHECK_RS2(); + + rs2_delete_sensor(sensor); + rs2_delete_sensor_list(sensors); + + return 0; +} + +/*! + * Process a frame as 6DOF data, does not assume ownership of the frame. + */ +static void +process_frame(struct rs_ddev *rs, rs2_frame *frame) +{ + rs2_error *e = NULL; + int ret = 0; + + ret = rs2_is_frame_extendable_to(frame, RS2_EXTENSION_POSE_FRAME, &e); + if (check_error(rs, e) != 0 || ret == 0) { + return; + } + + rs2_pose camera_pose; + rs2_pose_frame_get_pose_data(frame, &camera_pose, &e); + if (check_error(rs, e) != 0) { + return; + } + +#if 0 + rs2_timestamp_domain domain = rs2_get_frame_timestamp_domain(frame, &e); + if (check_error(rs, e) != 0) { + return; + } +#endif + + double timestamp_milliseconds = rs2_get_frame_timestamp(frame, &e); + if (check_error(rs, e) != 0) { + return; + } + + // Close enough + uint64_t now_real_ns = os_realtime_get_ns(); + uint64_t now_monotonic_ns = os_monotonic_get_ns(); + uint64_t timestamp_ns = (uint64_t)(timestamp_milliseconds * 1000.0 * 1000.0); + + // How far in the past is it? + uint64_t diff_ns = now_real_ns - timestamp_ns; + + // Adjust the timestamp to monotonic time. + timestamp_ns = now_monotonic_ns - diff_ns; + + /* + * Transfer the data to the struct. + */ + + struct xrt_space_relation relation; + + // Rotation/angular + relation.pose.orientation.x = camera_pose.rotation.x; + relation.pose.orientation.y = camera_pose.rotation.y; + relation.pose.orientation.z = camera_pose.rotation.z; + relation.pose.orientation.w = camera_pose.rotation.w; + relation.angular_velocity.x = camera_pose.angular_velocity.x; + relation.angular_velocity.y = camera_pose.angular_velocity.y; + relation.angular_velocity.z = camera_pose.angular_velocity.z; + + // Position/linear + relation.pose.position.x = camera_pose.translation.x; + relation.pose.position.y = camera_pose.translation.y; + relation.pose.position.z = camera_pose.translation.z; + relation.linear_velocity.x = camera_pose.velocity.x; + relation.linear_velocity.y = camera_pose.velocity.y; + relation.linear_velocity.z = camera_pose.velocity.z; + + // clang-format off + relation.relation_flags = + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | + XRT_SPACE_RELATION_POSITION_VALID_BIT | + XRT_SPACE_RELATION_LINEAR_VELOCITY_VALID_BIT | + XRT_SPACE_RELATION_ANGULAR_VELOCITY_VALID_BIT | + XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | + XRT_SPACE_RELATION_POSITION_TRACKED_BIT; + // clang-format on + + m_relation_history_push(rs->relation_hist, &relation, timestamp_ns); +} + +static int +update(struct rs_ddev *rs) +{ + rs2_frame *frames; + rs2_error *e = NULL; + + frames = rs2_pipeline_wait_for_frames(rs->rsc.pipeline, RS2_DEFAULT_TIMEOUT, &e); + if (check_error(rs, e) != 0) { + return 1; + } + + int num_frames = rs2_embedded_frames_count(frames, &e); + if (check_error(rs, e) != 0) { + return 1; + } + + for (int i = 0; i < num_frames; i++) { + rs2_frame *frame = rs2_extract_frame(frames, i, &e); + if (check_error(rs, e) != 0) { + rs2_release_frame(frames); + return 1; + } + + // Does not assume ownership of the frame. + process_frame(rs, frame); + rs2_release_frame(frame); + + rs2_release_frame(frames); + } + + return 0; +} + +static void * +rs_run_thread(void *ptr) +{ + struct rs_ddev *rs = (struct rs_ddev *)ptr; + + os_thread_helper_lock(&rs->oth); + + while (os_thread_helper_is_running_locked(&rs->oth)) { + + os_thread_helper_unlock(&rs->oth); + + int ret = update(rs); + if (ret < 0) { + return NULL; + } + } + + return NULL; +} + +static bool +load_config(struct rs_ddev *rs) +{ + struct u_config_json config_json = {0}; + + u_config_json_open_or_create_main_file(&config_json); + if (!config_json.file_loaded) { + return false; + } + + const cJSON *realsense_config_json = u_json_get(config_json.root, "config_realsense_ddev"); + if (realsense_config_json == NULL) { + return false; + } + + const cJSON *mapping = u_json_get(realsense_config_json, "enable_mapping"); + const cJSON *pose_jumping = u_json_get(realsense_config_json, "enable_pose_jumping"); + const cJSON *relocalization = u_json_get(realsense_config_json, "enable_relocalization"); + const cJSON *pose_prediction = u_json_get(realsense_config_json, "enable_pose_prediction"); + const cJSON *pose_filtering = u_json_get(realsense_config_json, "enable_pose_filtering"); + + // if json key isn't in the json, default to true. if it is in there, use json value + if (mapping != NULL) { + rs->enable_mapping = cJSON_IsTrue(mapping); + } + if (pose_jumping != NULL) { + rs->enable_pose_jumping = cJSON_IsTrue(pose_jumping); + } + if (relocalization != NULL) { + rs->enable_relocalization = cJSON_IsTrue(relocalization); + } + if (pose_prediction != NULL) { + rs->enable_pose_prediction = cJSON_IsTrue(pose_prediction); + } + if (pose_filtering != NULL) { + rs->enable_pose_filtering = cJSON_IsTrue(pose_filtering); + } + + cJSON_Delete(config_json.root); + + return true; +} + + +/* + * + * Device functions. + * + */ + +static void +rs_ddev_update_inputs(struct xrt_device *xdev) +{ + // Empty +} + +static void +rs_ddev_get_tracked_pose(struct xrt_device *xdev, + enum xrt_input_name name, + uint64_t at_timestamp_ns, + struct xrt_space_relation *out_relation) +{ + struct rs_ddev *rs = rs_ddev(xdev); + + if (name != XRT_INPUT_GENERIC_TRACKER_POSE) { + U_LOG_E("unknown input name"); + return; + } + + m_relation_history_get(rs->relation_hist, out_relation, at_timestamp_ns); +} + +static void +rs_ddev_get_view_pose(struct xrt_device *xdev, + const struct xrt_vec3 *eye_relation, + uint32_t view_index, + struct xrt_pose *out_pose) +{ + assert(false); +} + +static void +rs_ddev_destroy(struct xrt_device *xdev) +{ + struct rs_ddev *rs = rs_ddev(xdev); + + // Destroy the thread object. + os_thread_helper_destroy(&rs->oth); + + close_ddev(rs); + + free(rs); +} + + +/* + * + * 'Exported' functions. + * + */ + +struct xrt_device * +rs_ddev_create(int device_idx) +{ + struct rs_ddev *rs = U_DEVICE_ALLOCATE(struct rs_ddev, U_DEVICE_ALLOC_TRACKING_NONE, 1, 0); + + m_relation_history_create(&rs->relation_hist); + + rs->enable_mapping = true; + rs->enable_pose_jumping = true; + rs->enable_relocalization = true; + rs->enable_pose_prediction = true; + rs->enable_pose_filtering = true; + + if (load_config(rs)) { + U_LOG_D("Used config file"); + } else { + U_LOG_D("Did not use config file"); + } + + U_LOG_D("Realsense opts are %i %i %i %i %i\n", rs->enable_mapping, rs->enable_pose_jumping, + rs->enable_relocalization, rs->enable_pose_prediction, rs->enable_pose_filtering); + rs->base.update_inputs = rs_ddev_update_inputs; + rs->base.get_tracked_pose = rs_ddev_get_tracked_pose; + rs->base.get_view_pose = rs_ddev_get_view_pose; + rs->base.destroy = rs_ddev_destroy; + rs->base.name = XRT_DEVICE_REALSENSE; + rs->base.tracking_origin->type = XRT_TRACKING_TYPE_EXTERNAL_SLAM; + + // Print name. + snprintf(rs->base.str, XRT_DEVICE_NAME_LEN, "Intel RealSense Device-SLAM"); + snprintf(rs->base.serial, XRT_DEVICE_NAME_LEN, "Intel RealSense Device-SLAM"); + + rs->base.inputs[0].name = XRT_INPUT_GENERIC_TRACKER_POSE; + + int ret = 0; + + + + // Thread and other state. + ret = os_thread_helper_init(&rs->oth); + if (ret != 0) { + U_LOG_E("Failed to init threading!"); + rs_ddev_destroy(&rs->base); + return NULL; + } + + ret = create_ddev(rs, device_idx); + if (ret != 0) { + rs_ddev_destroy(&rs->base); + return NULL; + } + + ret = os_thread_helper_start(&rs->oth, rs_run_thread, rs); + if (ret != 0) { + U_LOG_E("Failed to start thread!"); + rs_ddev_destroy(&rs->base); + return NULL; + } + + rs->base.orientation_tracking_supported = true; + rs->base.position_tracking_supported = true; + rs->base.device_type = XRT_DEVICE_TYPE_GENERIC_TRACKER; + + return &rs->base; +} diff --git a/src/xrt/drivers/realsense/rs_driver.h b/src/xrt/drivers/realsense/rs_driver.h new file mode 100644 index 000000000..866841e76 --- /dev/null +++ b/src/xrt/drivers/realsense/rs_driver.h @@ -0,0 +1,90 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Internal header for the RealSense driver. + * @author Mateo de Mayo + * @ingroup drv_rs + */ +#pragma once + +#include "xrt/xrt_prober.h" + +#include "rs_interface.h" + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * @addtogroup drv_rs + * @{ + */ + +//! Container to store and manage useful objects from the RealSense API +struct rs_container +{ + rs2_error *error_status; + + // Used by prober and devices + rs2_context *context; //!< RealSense API context + rs2_device_list *device_list; //!< List of connected RealSense devices + int device_count; //!< Length of device_list + + // Used by devices + int device_idx; //!< `device` index in `device_list` + rs2_device *device; //!< Main device + rs2_pipeline *pipeline; //!< RealSense running pipeline + rs2_config *config; //!< Pipeline streaming configuration + rs2_pipeline_profile *profile; //!< Pipeline profile +}; + +//! Cleans up an @ref rs_container and resets its fields to NULL; +static void +rs_container_cleanup(struct rs_container *rsc) +{ + // A note about what is and what is not being deleted: + // In its documentation, the RealSense API specifies which calls require the + // caller to delete the returned object afterwards. By looking at the code of + // the API it seems that when that is not explicitly pointed out in the + // interface documentation, you should *not* delete the returned object. + + // clang-format off + if (rsc->profile) rs2_delete_pipeline_profile(rsc->profile); + if (rsc->config) rs2_delete_config(rsc->config); + if (rsc->pipeline) rs2_delete_pipeline(rsc->pipeline); + if (rsc->device) rs2_delete_device(rsc->device); + if (rsc->device_list) rs2_delete_device_list(rsc->device_list); + if (rsc->context) rs2_delete_context(rsc->context); + if (rsc->error_status) rs2_free_error(rsc->error_status); + // clang-format on + + rsc->profile = NULL; + rsc->config = NULL; + rsc->pipeline = NULL; + rsc->device = NULL; + rsc->device_idx = -1; + rsc->device_count = 0; + rsc->device_list = NULL; + rsc->context = NULL; + rsc->error_status = NULL; +} + +//! Create a RealSense device tracked with device-SLAM (T26x). +struct xrt_device * +rs_ddev_create(int device_idx); + +//! Create RealSense device tracked with host-SLAM (one with camera and IMU streams) +struct xrt_device * +rs_hdev_create(struct xrt_prober *xp, int device_idx); + +/*! + * @} + */ + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/realsense/rs_hdev.c b/src/xrt/drivers/realsense/rs_hdev.c new file mode 100644 index 000000000..78c069b02 --- /dev/null +++ b/src/xrt/drivers/realsense/rs_hdev.c @@ -0,0 +1,1012 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief RealSense device tracked with host-SLAM. + * @author Mateo de Mayo + * @ingroup drv_rs + * + * Originally created and tried on the D455 model but should work in any + * RealSense device that has video and IMU streams. + * + * Be aware that you need to properly set the SLAM_CONFIG file to match + * your camera specifics (stereo/mono, intrinsics, extrinsics, etc). + */ + +#include "math/m_filter_fifo.h" +#include "math/m_space.h" +#include "os/os_time.h" +#include "util/u_device.h" +#include "util/u_logging.h" +#include "util/u_debug.h" +#include "util/u_var.h" +#include "util/u_sink.h" +#include "util/u_config_json.h" +#include "util/u_format.h" +#include "tracking/t_tracking.h" + +#include "xrt/xrt_device.h" +#include "xrt/xrt_frameserver.h" +#include "xrt/xrt_frame.h" +#include "xrt/xrt_prober.h" +#include "xrt/xrt_config_have.h" + +#include "rs_driver.h" + +#include +#include +#include +#include + +// These defaults come from a D455 camera, they might not work for other devices +#define DEFAULT_STEREO true +#define DEFAULT_XRT_VIDEO_FORMAT XRT_FORMAT_L8 +#define DEFAULT_VIDEO_FORMAT RS2_FORMAT_Y8 +#define DEFAULT_VIDEO_WIDTH 640 +#define DEFAULT_VIDEO_HEIGHT 360 +#define DEFAULT_VIDEO_FPS 30 +#define DEFAULT_GYRO_FPS 200 +#define DEFAULT_ACCEL_FPS 250 +#define DEFAULT_STREAM_TYPE RS2_STREAM_INFRARED +#define DEFAULT_STREAM1_INDEX 1 +#define DEFAULT_STREAM2_INDEX 2 + +#define DEVICE_STRING "Intel RealSense Host-SLAM" + +#define RS_TRACE(...) U_LOG_XDEV_IFL_T(&rs->xdev, rs->ll, __VA_ARGS__) +#define RS_DEBUG(...) U_LOG_XDEV_IFL_D(&rs->xdev, rs->ll, __VA_ARGS__) +#define RS_INFO(...) U_LOG_XDEV_IFL_I(&rs->xdev, rs->ll, __VA_ARGS__) +#define RS_WARN(...) U_LOG_XDEV_IFL_W(&rs->xdev, rs->ll, __VA_ARGS__) +#define RS_ERROR(...) U_LOG_XDEV_IFL_E(&rs->xdev, rs->ll, __VA_ARGS__) +#define RS_ASSERT(predicate, ...) \ + do { \ + bool p = predicate; \ + if (!p) { \ + U_LOG(U_LOGGING_ERROR, __VA_ARGS__); \ + assert(false && "RS_ASSERT failed: " #predicate); \ + exit(EXIT_FAILURE); \ + } \ + } while (false); +#define RS_ASSERT_(predicate) RS_ASSERT(predicate, "Assertion failed " #predicate) + +// Debug assertions, not vital but useful for finding errors +#ifdef NDEBUG +#define RS_DASSERT(predicate, ...) +#define RS_DASSERT_(predicate) +#else +#define RS_DASSERT(predicate, ...) RS_ASSERT(predicate, __VA_ARGS__) +#define RS_DASSERT_(predicate) RS_ASSERT_(predicate) +#endif + +//! Utility for realsense API calls that can produce errors +#define DO(call, ...) \ + call(__VA_ARGS__, &rs->rsc.error_status); \ + check_error(rs, rs->rsc.error_status, __FILE__, __LINE__) + +//! Alternative to DO() with no arguments +#define DO_(call) \ + call(&rs->rsc.error_status); \ + check_error(rs, rs->rsc.error_status, __FILE__, __LINE__) + +//! @todo Use one RS_LOG option for the entire driver +DEBUG_GET_ONCE_LOG_OPTION(rs_log, "RS_HDEV_LOG", U_LOGGING_WARN) + +// Forward declarations +static void +receive_left_frame(struct xrt_frame_sink *sink, struct xrt_frame *); +static void +receive_right_frame(struct xrt_frame_sink *sink, struct xrt_frame *); +static void +receive_imu_sample(struct xrt_imu_sink *sink, struct xrt_imu_sample *); +static void +rs_hdev_node_break_apart(struct xrt_frame_node *); +static void +rs_hdev_node_destroy(struct xrt_frame_node *); + +/*! + * Host-SLAM tracked RealSense device (any RealSense device with camera and IMU streams). + * + * @implements xrt_device + * @implements xrt_fs + * @implements xrt_frame_node + */ +struct rs_hdev +{ + struct xrt_device xdev; + struct xrt_frame_context xfctx; + struct xrt_frame_node node; + struct xrt_fs xfs; + struct xrt_tracked_slam *slam; + struct xrt_pose pose; //!< Device pose + struct xrt_pose offset; //!< Additional offset to apply to `pose` + enum u_logging_level ll; //!< Log level + + // Sinks + struct xrt_frame_sink left_sink; //!< Intermediate sink for left camera frames + struct xrt_frame_sink right_sink; //!< Intermediate sink for right camera frames + struct xrt_imu_sink imu_sink; //!< Intermediate sink for IMU samples + struct xrt_slam_sinks in_sinks; //!< Pointers to intermediate sinks + struct xrt_slam_sinks out_sinks; //!< Pointers to downstream sinks + + // UI Sinks + struct u_sink_debug ui_left_sink; //!< Sink to display left frames in UI + struct u_sink_debug ui_right_sink; //!< Sink to display right frames in UI + struct m_ff_vec3_f32 *gyro_ff; //!< Queue of gyroscope data to display in UI + struct m_ff_vec3_f32 *accel_ff; //!< Queue of accelerometer data to display in UI + + struct rs_container rsc; //!< Container of RealSense API objects + + // Properties loaded from json file and used when configuring the realsense pipeline + bool stereo; //!< Indicates whether to use one or two cameras + rs2_format video_format; //!< Indicates desired frame color format + enum xrt_format xrt_video_format; //!< corresponding format for video_format + int video_width; //!< Indicates desired frame width + int video_height; //!< Indicates desired frame height + int video_fps; //!< Indicates desired fps + int gyro_fps; //!< Indicates desired gyroscope samples per second + int accel_fps; //!< Indicates desired accelerometer samples per second + rs2_stream stream_type; //!< Indicates desired stream type for the cameras + int stream1_index; //!< Indicates desired stream index for first stream + int stream2_index; //!< Indicates desired stream index for second stream + + bool is_running; //!< Whether the device is streaming + + //! Very simple struct to merge the two acc/gyr streams into one IMU stream. + //! It just pushes on every gyro sample and reuses the latest acc sample. + struct + { + struct os_mutex mutex; //!< Gyro and accel come from separate threads + struct xrt_vec3 accel; //!< Last received accelerometer values + struct xrt_vec3 gyro; //!< Last received gyroscope values + } partial_imu_sample; +}; + +//! @todo Unify check_error() and DO() usage thorough the driver. +static bool +check_error(struct rs_hdev *rs, rs2_error *e, const char *file, int line) +{ + if (e == NULL) { + return false; // No errors + } + + RS_ERROR("rs_error was raised when calling %s(%s):", rs2_get_failed_function(e), rs2_get_failed_args(e)); + RS_ERROR("%s:%d: %s", file, line, rs2_get_error_message(e)); + exit(EXIT_FAILURE); +} + + +/* + * + * Device functionality + * + */ + +static inline struct rs_hdev * +rs_hdev_from_xdev(struct xrt_device *xdev) +{ + struct rs_hdev *rs = container_of(xdev, struct rs_hdev, xdev); + return rs; +} + +static void +rs_hdev_update_inputs(struct xrt_device *xdev) +{ + return; +} + +//! Specific pose corrections for Kimera and the D455 camera +XRT_MAYBE_UNUSED static inline struct xrt_pose +rs_hdev_correct_pose_from_kimera(struct xrt_pose pose) +{ + // Correct swapped axes + struct xrt_pose swapped = {0}; + swapped.position.x = -pose.position.y; + swapped.position.y = -pose.position.z; + swapped.position.z = pose.position.x; + swapped.orientation.x = -pose.orientation.y; + swapped.orientation.y = -pose.orientation.z; + swapped.orientation.z = pose.orientation.x; + swapped.orientation.w = pose.orientation.w; + + // Correct orientation + //! @todo Encode this transformation into constants + struct xrt_space_relation out_relation; + struct xrt_space_graph space_graph = {0}; + struct xrt_pose pre_correction = {{-0.5, -0.5, -0.5, 0.5}, {0, 0, 0}}; // euler(90, 90, 0) + float sin45 = 0.7071067811865475; + struct xrt_pose pos_correction = {{sin45, 0, sin45, 0}, {0, 0, 0}}; // euler(180, 90, 0) + m_space_graph_add_pose(&space_graph, &pre_correction); + m_space_graph_add_pose(&space_graph, &swapped); + m_space_graph_add_pose(&space_graph, &pos_correction); + m_space_graph_resolve(&space_graph, &out_relation); + return out_relation.pose; +} + +static void +rs_hdev_get_tracked_pose(struct xrt_device *xdev, + enum xrt_input_name name, + uint64_t at_timestamp_ns, + struct xrt_space_relation *out_relation) +{ + struct rs_hdev *rs = rs_hdev_from_xdev(xdev); + RS_ASSERT_(rs->slam != NULL); + RS_ASSERT_(at_timestamp_ns < INT64_MAX); + + xrt_tracked_slam_get_tracked_pose(rs->slam, at_timestamp_ns, out_relation); + + int pose_bits = XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT; + bool pose_tracked = out_relation->relation_flags & pose_bits; + + if (pose_tracked) { +#if defined(XRT_HAVE_KIMERA_SLAM) + rs->pose = rs_hdev_correct_pose_from_kimera(out_relation->pose); +#else + rs->pose = out_relation->pose; +#endif + } + + struct xrt_space_graph space_graph = {0}; + m_space_graph_add_pose(&space_graph, &rs->pose); + m_space_graph_add_pose(&space_graph, &rs->offset); + m_space_graph_resolve(&space_graph, out_relation); + out_relation->relation_flags = (enum xrt_space_relation_flags)( + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_POSITION_VALID_BIT | + XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT); +} + +static void +rs_hdev_destroy(struct xrt_device *xdev) +{ + struct rs_hdev *rs = rs_hdev_from_xdev(xdev); + RS_INFO("Destroying rs_hdev"); + xrt_frame_context_destroy_nodes(&rs->xfctx); + RS_ASSERT(!rs->is_running, "Trying to destroy device while it is still streaming"); + u_device_free(&rs->xdev); +} + + +/* + * + * JSON functionality + * + */ + +#define JSON_CONFIG_FIELD_NAME "config_realsense_hdev" + +//! Helper function for loading an int field from a json container and printing useful +//! messages along it. *out is expected to come preloaded with a default value. +static void +json_int(struct rs_hdev *rs, const cJSON *json, const char *field, int *out) +{ + if (!u_json_get_int(u_json_get(json, field), out)) { + // This is a warning because we want the user to set these config fields + RS_WARN("Using default %s.%s=%d", JSON_CONFIG_FIELD_NAME, field, *out); + } else { + RS_DEBUG("Using %s.%s=%d", JSON_CONFIG_FIELD_NAME, field, *out); + } +} + +//! Similar to @ref json_int but for bools. +static void +json_bool(struct rs_hdev *rs, const cJSON *json, const char *field, bool *out) +{ + if (!u_json_get_bool(u_json_get(json, field), out)) { + // This is a warning because we want the user to set these config fields + RS_WARN("Using default %s.%s=%s", JSON_CONFIG_FIELD_NAME, field, *out ? "true" : "false"); + } else { + RS_DEBUG("Using %s.%s=%s", JSON_CONFIG_FIELD_NAME, field, *out ? "true" : "false"); + } +} + +//! Similar to @ref json_int but for a video rs2_format, also sets the +//! equivalent xrt_format if any. +static void +json_rs2_format( + struct rs_hdev *rs, const cJSON *json, const char *field, rs2_format *out_rformat, enum xrt_format *out_xformat) +{ + int format_int = (int)*out_rformat; + bool valid_field = u_json_get_int(u_json_get(json, field), &format_int); + if (!valid_field) { + RS_WARN("Using default %s.%s=%d (%d)", JSON_CONFIG_FIELD_NAME, field, *out_rformat, *out_xformat); + return; + } + + rs2_format rformat = (rs2_format)format_int; + enum xrt_format xformat; + if (rformat == RS2_FORMAT_Y8) { + xformat = XRT_FORMAT_L8; + } else if (rformat == RS2_FORMAT_RGB8 || rformat == RS2_FORMAT_BGR8) { + xformat = XRT_FORMAT_R8G8B8; + } else { + RS_ERROR("Invalid %s.%s=%d", JSON_CONFIG_FIELD_NAME, field, format_int); + RS_ERROR("Valid values: %d, %d, %d", RS2_FORMAT_Y8, RS2_FORMAT_RGB8, RS2_FORMAT_BGR8); + RS_ERROR("Using default %s.%s=%d (%d)", JSON_CONFIG_FIELD_NAME, field, *out_rformat, *out_xformat); + + // Reaching this doesn't mean that a matching xrt_format doesn't exist, just + // that I didn't need it. Feel free to add it. + + return; + } + + *out_rformat = rformat; + *out_xformat = xformat; + RS_DEBUG("Using %s.%s=%d (xrt_format=%d)", JSON_CONFIG_FIELD_NAME, field, *out_rformat, *out_xformat); +} + +//! Similar to @ref json_int but for a rs2_stream type. +static void +json_rs2_stream(struct rs_hdev *rs, const cJSON *json, const char *field, rs2_stream *out_stream) +{ + int stream_int = (int)*out_stream; + bool valid_field = u_json_get_int(u_json_get(json, field), &stream_int); + if (!valid_field) { + RS_WARN("Using default %s.%s=%d", JSON_CONFIG_FIELD_NAME, field, *out_stream); + return; + } + + rs2_stream rstream = (rs2_stream)stream_int; + if (rstream != RS2_STREAM_COLOR && rstream != RS2_STREAM_INFRARED && rstream != RS2_STREAM_FISHEYE) { + RS_ERROR("Invalid %s.%s=%d", JSON_CONFIG_FIELD_NAME, field, stream_int); + RS_ERROR("Valid values: %d, %d, %d", RS2_STREAM_COLOR, RS2_STREAM_INFRARED, RS2_STREAM_FISHEYE); + RS_ERROR("Using default %s.%s=%d", JSON_CONFIG_FIELD_NAME, field, *out_stream); + return; + } + + *out_stream = rstream; + RS_DEBUG("Using %s.%s=%d", JSON_CONFIG_FIELD_NAME, field, *out_stream); +} + +static void +rs_hdev_load_stream_options_from_json(struct rs_hdev *rs) +{ + // Set default values + rs->stereo = DEFAULT_STEREO; + rs->xrt_video_format = DEFAULT_XRT_VIDEO_FORMAT; + rs->video_format = DEFAULT_VIDEO_FORMAT; + rs->video_width = DEFAULT_VIDEO_WIDTH; + rs->video_height = DEFAULT_VIDEO_HEIGHT; + rs->video_fps = DEFAULT_VIDEO_FPS; + rs->gyro_fps = DEFAULT_GYRO_FPS; + rs->accel_fps = DEFAULT_ACCEL_FPS; + rs->stream_type = DEFAULT_STREAM_TYPE; + rs->stream1_index = DEFAULT_STREAM1_INDEX; + rs->stream2_index = DEFAULT_STREAM2_INDEX; + + struct u_config_json config = {0}; + u_config_json_open_or_create_main_file(&config); + if (!config.file_loaded) { + RS_WARN("Unable to load config file, will use default stream values"); + cJSON_Delete(config.root); + return; + } + + const cJSON *hdev_config = u_json_get(config.root, JSON_CONFIG_FIELD_NAME); + if (hdev_config == NULL) { + RS_WARN("Field '%s' was not present in the json file, will use default values", JSON_CONFIG_FIELD_NAME); + } + + json_bool(rs, hdev_config, "stereo", &rs->stereo); + json_rs2_format(rs, hdev_config, "video_format", &rs->video_format, &rs->xrt_video_format); + json_int(rs, hdev_config, "video_width", &rs->video_width); + json_int(rs, hdev_config, "video_height", &rs->video_height); + json_int(rs, hdev_config, "video_fps", &rs->video_fps); + json_int(rs, hdev_config, "gyro_fps", &rs->gyro_fps); + json_int(rs, hdev_config, "accel_fps", &rs->accel_fps); + json_rs2_stream(rs, hdev_config, "stream_type", &rs->stream_type); + json_int(rs, hdev_config, "stream1_index", &rs->stream1_index); + json_int(rs, hdev_config, "stream2_index", &rs->stream2_index); + + cJSON_Delete(config.root); +} + + +/* + * + * Realsense functionality + * + */ + +//! Disable any laser emitters because they confuse SLAM feature detection +static void +disable_all_laser_emitters(struct rs_hdev *rs) +{ + struct rs_container *rsc = &rs->rsc; + rs2_sensor_list *sensors = DO(rs2_query_sensors, rsc->device); + int sensors_count = DO(rs2_get_sensors_count, sensors); + for (int i = 0; i < sensors_count; i++) { + rs2_sensor *sensor = DO(rs2_create_sensor, sensors, i); + rs2_options *sensor_options = (rs2_options *)sensor; + bool has_emitter = DO(rs2_supports_option, sensor_options, RS2_OPTION_EMITTER_ENABLED); + if (has_emitter) { + DO(rs2_set_option, sensor_options, RS2_OPTION_EMITTER_ENABLED, 0); + } + rs2_delete_sensor(sensor); + } + rs2_delete_sensor_list(sensors); +} + + +/* + * + * Stream functionality + * + */ + +static void +rs_hdev_frame_destroy(struct xrt_frame *xf) +{ + rs2_frame *rframe = (rs2_frame *)xf->owner; + rs2_release_frame(rframe); + free(xf); +} + +static void +rs2xrt_frame(struct rs_hdev *rs, rs2_frame *rframe, struct xrt_frame **out_xframe) +{ + RS_ASSERT_(*out_xframe == NULL); + + uint64_t number = DO(rs2_get_frame_number, rframe); + double timestamp_ms = DO(rs2_get_frame_timestamp, rframe); + uint8_t *data = (uint8_t *)DO(rs2_get_frame_data, rframe); + int bytes_per_pixel = u_format_block_size(rs->xrt_video_format); + int stride = rs->video_width * bytes_per_pixel; + +#ifndef NDEBUG // Debug only: check that the realsense stream is behaving as expected + bool is_video_frame = DO(rs2_is_frame_extendable_to, rframe, RS2_EXTENSION_VIDEO_FRAME); + int rs_bits_per_pixel = DO(rs2_get_frame_bits_per_pixel, rframe); + int rs_width = DO(rs2_get_frame_width, rframe); + int rs_height = DO(rs2_get_frame_height, rframe); + int rs_stride = DO(rs2_get_frame_stride_in_bytes, rframe); + RS_DASSERT_(is_video_frame); + RS_DASSERT_(rs_bits_per_pixel == bytes_per_pixel * 8); + RS_DASSERT(rs_width == rs->video_width, "%d != %d", rs_width, rs->video_width); + RS_DASSERT(rs_height == rs->video_height, "%d != %d", rs_height, rs->video_height); + RS_DASSERT(rs_stride == stride, "%d != %d", rs_stride, stride); +#endif + + struct xrt_frame *xf = U_TYPED_CALLOC(struct xrt_frame); + xf->reference.count = 1; + xf->destroy = rs_hdev_frame_destroy; + xf->owner = rframe; + xf->width = rs->video_width; + xf->height = rs->video_height; + xf->stride = stride; + xf->size = rs->video_height * stride; + xf->data = data; + + xf->format = rs->xrt_video_format; + xf->stereo_format = XRT_STEREO_FORMAT_NONE; //!< @todo Use a stereo xrt_format + + uint64_t timestamp_ns = timestamp_ms * 1000 * 1000; + xf->timestamp = timestamp_ns; + xf->source_timestamp = timestamp_ns; + xf->source_sequence = number; + xf->source_id = rs->xfs.source_id; + + *out_xframe = xf; +} + +static void +handle_frameset(struct rs_hdev *rs, rs2_frame *frames) +{ + + // Check number of frames on debug builds + int num_of_frames = DO(rs2_embedded_frames_count, frames); + if (rs->stereo) { + RS_DASSERT(num_of_frames == 2, "Stereo frameset contains %d (!= 2) frames", num_of_frames); + } else { + RS_DASSERT(num_of_frames == 1, "Non-stereo frameset contains %d (!= 1) frames", num_of_frames); + } + + // Left frame + rs2_frame *rframe_left = DO(rs2_extract_frame, frames, 0); + struct xrt_frame *xf_left = NULL; + rs2xrt_frame(rs, rframe_left, &xf_left); + + if (rs->stereo) { + + // Right frame + rs2_frame *rframe_right = DO(rs2_extract_frame, frames, 1); + struct xrt_frame *xf_right = NULL; + rs2xrt_frame(rs, rframe_right, &xf_right); + + if (xf_left->timestamp == xf_right->timestamp) { + xrt_sink_push_frame(rs->in_sinks.left, xf_left); + xrt_sink_push_frame(rs->in_sinks.right, xf_right); + } else { + // This usually happens only once at start and never again + RS_WARN("Realsense device sent left and right frames with different timestamps %ld != %ld", + xf_left->timestamp, xf_right->timestamp); + } + + xrt_frame_reference(&xf_right, NULL); + } else { + xrt_sink_push_frame(rs->in_sinks.left, xf_left); + } + + xrt_frame_reference(&xf_left, NULL); + + // Release frameset but individual frames will be released on xrt_frame destruction + rs2_release_frame(frames); +} + +//! Decides when to submit the full IMU sample out of separate +//! gyroscope/accelerometer samples. +static void +partial_imu_sample_push(struct rs_hdev *rs, timepoint_ns ts, struct xrt_vec3 vals, bool is_gyro) +{ + os_mutex_lock(&rs->partial_imu_sample.mutex); + + if (is_gyro) { + rs->partial_imu_sample.gyro = vals; + } else { + rs->partial_imu_sample.accel = vals; + } + struct xrt_vec3 gyro = rs->partial_imu_sample.gyro; + struct xrt_vec3 accel = rs->partial_imu_sample.accel; + + // Push IMU sample from fastest motion sensor arrives, reuse latest data from the other sensor (or zero) + bool should_submit = (rs->gyro_fps > rs->accel_fps) == is_gyro; + if (should_submit) { + struct xrt_imu_sample sample = {ts, {accel.x, accel.y, accel.z}, {gyro.x, gyro.y, gyro.z}}; + xrt_sink_push_imu(rs->in_sinks.imu, &sample); + } + + os_mutex_unlock(&rs->partial_imu_sample.mutex); +} + +static void +handle_gyro_frame(struct rs_hdev *rs, rs2_frame *frame) +{ + const float *data = DO(rs2_get_frame_data, frame); + +#ifndef NDEBUG + int data_size = DO(rs2_get_frame_data_size, frame); + RS_DASSERT(data_size == 3 * sizeof(float) || data_size == 4 * sizeof(float), "Unexpected size=%d", data_size); + RS_DASSERT_(data_size != 4 || data[3] == 0); +#endif + + double timestamp_ms = DO(rs2_get_frame_timestamp, frame); + timepoint_ns timestamp_ns = timestamp_ms * 1000 * 1000; + struct xrt_vec3 gyro = {data[0], data[1], data[2]}; + RS_TRACE("gyro t=%ld x=%f y=%f z=%f", timestamp_ns, gyro.x, gyro.y, gyro.z); + partial_imu_sample_push(rs, timestamp_ns, gyro, true); + rs2_release_frame(frame); +} + +static void +handle_accel_frame(struct rs_hdev *rs, rs2_frame *frame) +{ + const float *data = DO(rs2_get_frame_data, frame); + +#ifndef NDEBUG + int data_size = DO(rs2_get_frame_data_size, frame); + // For some strange reason data_size is 4 for samples that can use hardware + // timestamps. And that last element data[3] seems to always be zero. + RS_DASSERT(data_size == 3 * sizeof(float) || data_size == 4 * sizeof(float), "Unexpected size=%d", data_size); + RS_DASSERT_(data_size != 4 || data[3] == 0); +#endif + + double timestamp_ms = DO(rs2_get_frame_timestamp, frame); + timepoint_ns timestamp_ns = timestamp_ms * 1000 * 1000; + struct xrt_vec3 accel = {data[0], data[1], data[2]}; + RS_TRACE("accel t=%ld x=%f y=%f z=%f", timestamp_ns, accel.x, accel.y, accel.z); + partial_imu_sample_push(rs, timestamp_ns, accel, false); + rs2_release_frame(frame); +} + +//! Checks that the timestamp domain of the realsense sample (the frame) is in +//! global time or, at the very least, in another domain that we support. +static inline void +check_global_time(struct rs_hdev *rs, rs2_frame *frame, rs2_stream stream_type) +{ + +#ifndef NDEBUG // Check valid timestamp domains only on debug builds + rs2_timestamp_domain ts_domain = DO(rs2_get_frame_timestamp_domain, frame); + bool using_global_time = ts_domain == RS2_TIMESTAMP_DOMAIN_GLOBAL_TIME; + bool acceptable_timestamp_domain = using_global_time; + + //! @note We should be ensuring that we have the same timestamp domains in all + //! sensors. But the user might have a newer kernel versions that is not + //! supported by the RealSense DKMS package that allows GLOBAL_TIME for all + //! sensors. From my experience and based on other users' reports, the only + //! affected sensor without GLOBAL_TIME is the gyroscope, which is ~30ms off. + //! See https://github.com/IntelRealSense/librealsense/issues/5710 + + bool is_accel = stream_type == RS2_STREAM_ACCEL; + bool is_gyro = stream_type == RS2_STREAM_GYRO; + bool is_motion_sensor = is_accel || is_gyro; + + if (is_motion_sensor) { + bool is_gyro_slower = rs->gyro_fps < rs->accel_fps; + bool is_slower_motion_sensor = is_gyro_slower == is_gyro; + + // We allow different domains for the slower sensor because partial_imu_sample + // discards those timestamps + acceptable_timestamp_domain |= is_slower_motion_sensor; + } + + if (!acceptable_timestamp_domain) { + RS_ERROR("Invalid ts_domain=%s", rs2_timestamp_domain_to_string(ts_domain)); + RS_ERROR("One of your RealSense sensors is not using GLOBAL_TIME domain for its timestamps."); + RS_ERROR("This should be solved by applying the kernel patch required by the RealSense SDK."); + if (is_motion_sensor) { + const char *a = is_accel ? "accelerometer" : "gyroscope"; + const char *b = is_accel ? "gyroscope" : "accelerometer"; + RS_ERROR("As an alternative, set %s frequency to be greater than %s frequency.", b, a); + } + RS_DASSERT(false, "Unacceptable timestamp domain %s", rs2_timestamp_domain_to_string(ts_domain)); + } +#endif +} + +static void +on_frame(rs2_frame *frame, void *ptr) +{ + struct rs_hdev *rs = (struct rs_hdev *)ptr; + + const rs2_stream_profile *stream = DO(rs2_get_frame_stream_profile, frame); + rs2_stream stream_type; + rs2_format format; + int index, unique_id, framerate; + DO(rs2_get_stream_profile_data, stream, &stream_type, &format, &index, &unique_id, &framerate); + + bool is_frameset = DO(rs2_is_frame_extendable_to, frame, RS2_EXTENSION_COMPOSITE_FRAME); + bool is_motion_frame = DO(rs2_is_frame_extendable_to, frame, RS2_EXTENSION_MOTION_FRAME); + check_global_time(rs, frame, stream_type); + + if (stream_type == rs->stream_type) { + RS_DASSERT_(is_frameset && format == rs->video_format && + (index == rs->stream1_index || index == rs->stream2_index) && framerate == rs->video_fps); + handle_frameset(rs, frame); + } else if (stream_type == RS2_STREAM_GYRO) { + RS_DASSERT_(is_motion_frame && format == RS2_FORMAT_MOTION_XYZ32F && framerate == rs->gyro_fps); + handle_gyro_frame(rs, frame); + } else if (stream_type == RS2_STREAM_ACCEL) { + RS_DASSERT_(is_motion_frame && format == RS2_FORMAT_MOTION_XYZ32F && framerate == rs->accel_fps); + handle_accel_frame(rs, frame); + } else { + RS_ASSERT(false, "Unexpected stream"); + } +} + + +/* + * + * Frameserver functionality + * + */ + +static inline struct rs_hdev * +rs_hdev_from_xfs(struct xrt_fs *xfs) +{ + struct rs_hdev *rs = container_of(xfs, struct rs_hdev, xfs); + return rs; +} + +static bool +rs_hdev_enumerate_modes(struct xrt_fs *xfs, struct xrt_fs_mode **out_modes, uint32_t *out_count) +{ + //! @todo implement + RS_ASSERT(false, "Not Implemented"); + return false; +} + +static bool +rs_hdev_configure_capture(struct xrt_fs *xfs, struct xrt_fs_capture_parameters *cp) +{ + //! @todo implement + RS_ASSERT(false, "Not Implemented"); + return false; +} + +static bool +rs_hdev_stream_stop(struct xrt_fs *xfs) +{ + struct rs_hdev *rs = rs_hdev_from_xfs(xfs); + if (rs->is_running) { + DO(rs2_pipeline_stop, rs->rsc.pipeline); + rs->is_running = false; + } + return true; +} + +static bool +rs_hdev_is_running(struct xrt_fs *xfs) +{ + struct rs_hdev *rs = rs_hdev_from_xfs(xfs); + return rs->is_running; +} + +static bool +rs_hdev_stream_start(struct xrt_fs *xfs, + struct xrt_frame_sink *xs, + enum xrt_fs_capture_type capture_type, + uint32_t descriptor_index) +{ + struct rs_hdev *rs = rs_hdev_from_xfs(xfs); + if (xs == NULL && capture_type == XRT_FS_CAPTURE_TYPE_TRACKING) { + RS_ASSERT(rs->out_sinks.left != NULL, "No left sink provided"); + RS_INFO("Starting RealSense stream in tracking mode"); + } else if (xs != NULL && capture_type == XRT_FS_CAPTURE_TYPE_CALIBRATION) { + RS_INFO("Starting RealSense stream in calibration mode, will stream only left frames"); + rs->out_sinks.left = xs; + } else { + RS_ASSERT(false, "Unsupported stream configuration xs=%p capture_type=%d", (void *)xs, capture_type); + return false; + } + + struct rs_container *rsc = &rs->rsc; + rsc->profile = DO(rs2_pipeline_start_with_config_and_callback, rsc->pipeline, rsc->config, on_frame, rs); + + disable_all_laser_emitters(rs); + + rs->is_running = true; + return rs->is_running; +} + +static bool +rs_hdev_slam_stream_start(struct xrt_fs *xfs, struct xrt_slam_sinks *sinks) +{ + struct rs_hdev *rs = rs_hdev_from_xfs(xfs); + rs->out_sinks = *sinks; + return rs_hdev_stream_start(xfs, NULL, XRT_FS_CAPTURE_TYPE_TRACKING, 0); +} + +//! Create and open the frame server for IMU/camera streaming. +static struct xrt_fs * +rs_hdev_fs_create(struct rs_hdev *rs, int device_idx) +{ + // Setup xrt_fs + struct xrt_fs *xfs = &rs->xfs; + xfs->enumerate_modes = rs_hdev_enumerate_modes; + xfs->configure_capture = rs_hdev_configure_capture; + xfs->stream_start = rs_hdev_stream_start; + xfs->slam_stream_start = rs_hdev_slam_stream_start; + xfs->stream_stop = rs_hdev_stream_stop; + xfs->is_running = rs_hdev_is_running; + snprintf(xfs->name, sizeof(xfs->name), DEVICE_STRING " X"); + snprintf(xfs->product, sizeof(xfs->product), DEVICE_STRING " P"); + snprintf(xfs->manufacturer, sizeof(xfs->manufacturer), DEVICE_STRING " M"); + snprintf(xfs->serial, sizeof(xfs->serial), DEVICE_STRING " S"); + xfs->source_id = 0x2EA15E115E; + + // Setup realsense pipeline + struct rs_container *rsc = &rs->rsc; + rsc->error_status = NULL; + rsc->context = DO(rs2_create_context, RS2_API_VERSION); + rsc->device_list = DO(rs2_query_devices, rsc->context); + rsc->device_count = DO(rs2_get_device_count, rsc->device_list); + rsc->device_idx = device_idx; + rsc->device = DO(rs2_create_device, rsc->device_list, rsc->device_idx); + rsc->pipeline = DO(rs2_create_pipeline, rsc->context); + rsc->config = DO_(rs2_create_config); + + // Set the pipeline to start specifically on the realsense device the prober selected + bool hdev_has_serial = DO(rs2_supports_device_info, rsc->device, RS2_CAMERA_INFO_SERIAL_NUMBER); + if (hdev_has_serial) { + const char *hdev_serial = DO(rs2_get_device_info, rsc->device, RS2_CAMERA_INFO_SERIAL_NUMBER); + DO(rs2_config_enable_device, rsc->config, hdev_serial); + } else { + RS_WARN("Unexpected, the realsense device in use does not provide a serial number."); + } + + rs_hdev_load_stream_options_from_json(rs); + + rs2_stream stream_type = rs->stream_type; + int width = rs->video_width; + int height = rs->video_height; + int fps = rs->video_fps; + rs2_format format = rs->video_format; + DO(rs2_config_enable_stream, rsc->config, RS2_STREAM_GYRO, 0, 0, 0, RS2_FORMAT_MOTION_XYZ32F, rs->gyro_fps); + DO(rs2_config_enable_stream, rsc->config, RS2_STREAM_ACCEL, 0, 0, 0, RS2_FORMAT_MOTION_XYZ32F, rs->accel_fps); + DO(rs2_config_enable_stream, rsc->config, stream_type, rs->stream1_index, width, height, format, fps); + if (rs->stereo) { + DO(rs2_config_enable_stream, rsc->config, stream_type, rs->stream2_index, width, height, format, fps); + } + + // Setup sinks + rs->left_sink.push_frame = receive_left_frame; + rs->right_sink.push_frame = receive_right_frame; + rs->imu_sink.push_imu = receive_imu_sample; + rs->in_sinks.left = &rs->left_sink; + rs->in_sinks.right = &rs->right_sink; + rs->in_sinks.imu = &rs->imu_sink; + + // Setup UI + u_sink_debug_init(&rs->ui_left_sink); + u_sink_debug_init(&rs->ui_right_sink); + m_ff_vec3_f32_alloc(&rs->gyro_ff, 1000); + m_ff_vec3_f32_alloc(&rs->accel_ff, 1000); + u_var_add_root(rs, "RealSense Device", false); + u_var_add_ro_text(rs, "Host SLAM", "Tracked by"); + u_var_add_log_level(rs, &rs->ll, "Log Level"); + u_var_add_pose(rs, &rs->pose, "SLAM Pose"); + u_var_add_pose(rs, &rs->offset, "Offset Pose"); + u_var_add_ro_ff_vec3_f32(rs, rs->gyro_ff, "Gyroscope"); + u_var_add_ro_ff_vec3_f32(rs, rs->accel_ff, "Accelerometer"); + u_var_add_sink_debug(rs, &rs->ui_left_sink, "Left Camera"); + u_var_add_sink_debug(rs, &rs->ui_right_sink, "Right Camera"); + + // Setup node + struct xrt_frame_node *xfn = &rs->node; + xfn->break_apart = rs_hdev_node_break_apart; + xfn->destroy = rs_hdev_node_destroy; + xrt_frame_context_add(&rs->xfctx, &rs->node); + + // Setup IMU synchronizer lock + os_mutex_init(&rs->partial_imu_sample.mutex); + + return xfs; +} + + +/* + * + * SLAM tracker functionality + * + */ + +//! @todo The slam tracker should be started from the tracking factory, not from here +static bool +rs_hdev_slam_track(struct rs_hdev *rs, struct xrt_prober *xp) +{ +#ifdef XRT_HAVE_SLAM + struct xrt_slam_sinks *sinks = NULL; + + int create_status = t_slam_create(&rs->xfctx, &rs->slam, &sinks); + if (create_status != 0) { + return false; + } + + bool stream_started = xrt_fs_slam_stream_start(&rs->xfs, sinks); + if (!stream_started) { + return false; + } + + int start_status = t_slam_start(rs->slam); + if (start_status != 0) { + return false; + } + return true; +#else + return false; +#endif +} + + +/* + * + * Sinks functionality + * + */ + +static void +receive_left_frame(struct xrt_frame_sink *sink, struct xrt_frame *xf) +{ + struct rs_hdev *rs = container_of(sink, struct rs_hdev, left_sink); + RS_TRACE("left img t=%ld source_t=%ld", xf->timestamp, xf->source_timestamp); + u_sink_debug_push_frame(&rs->ui_left_sink, xf); + if (rs->out_sinks.left) { + xrt_sink_push_frame(rs->out_sinks.left, xf); + } +} + +static void +receive_right_frame(struct xrt_frame_sink *sink, struct xrt_frame *xf) +{ + struct rs_hdev *rs = container_of(sink, struct rs_hdev, right_sink); + RS_TRACE("right img t=%ld source_t=%ld", xf->timestamp, xf->source_timestamp); + u_sink_debug_push_frame(&rs->ui_right_sink, xf); + if (rs->out_sinks.right) { + xrt_sink_push_frame(rs->out_sinks.right, xf); + } +} + +static void +receive_imu_sample(struct xrt_imu_sink *sink, struct xrt_imu_sample *s) +{ + struct rs_hdev *rs = container_of(sink, struct rs_hdev, imu_sink); + + timepoint_ns ts = s->timestamp_ns; + struct xrt_vec3_f64 a = s->accel_m_s2; + struct xrt_vec3_f64 w = s->gyro_rad_secs; + RS_TRACE("imu t=%ld a=(%f %f %f) w=(%f %f %f)", ts, a.x, a.y, a.z, w.x, w.y, w.z); + + // Push to debug UI by adjusting the timestamp to monotonic time + + struct xrt_vec3 gyro = {(float)w.x, (float)w.y, (float)w.z}; + struct xrt_vec3 accel = {(float)a.x, (float)a.y, (float)a.z}; + uint64_t now_realtime = os_realtime_get_ns(); + uint64_t now_monotonic = os_monotonic_get_ns(); + RS_DASSERT_(now_realtime < INT64_MAX); + + // Assertion commented because GLOBAL_TIME makes ts be a bit in the future + // RS_DASSERT_(now_realtime < INT64_MAX && (timepoint_ns)now_realtime > ts); + + uint64_t imu_monotonic = now_monotonic - (now_realtime - ts); + m_ff_vec3_f32_push(rs->gyro_ff, &gyro, imu_monotonic); + m_ff_vec3_f32_push(rs->accel_ff, &accel, imu_monotonic); + + if (rs->out_sinks.imu) { + xrt_sink_push_imu(rs->out_sinks.imu, s); + } +} + + +/* + * + * Frame node functionality + * + */ + +static void +rs_hdev_node_break_apart(struct xrt_frame_node *node) +{ + struct rs_hdev *rs = container_of(node, struct rs_hdev, node); + rs_hdev_stream_stop(&rs->xfs); +} + +static void +rs_hdev_node_destroy(struct xrt_frame_node *node) +{ + struct rs_hdev *rs = container_of(node, struct rs_hdev, node); + os_mutex_destroy(&rs->partial_imu_sample.mutex); + u_var_remove_root(rs); + u_sink_debug_destroy(&rs->ui_left_sink); + u_sink_debug_destroy(&rs->ui_right_sink); + m_ff_vec3_f32_free(&rs->gyro_ff); + m_ff_vec3_f32_free(&rs->accel_ff); + + rs_container_cleanup(&rs->rsc); + + RS_INFO("Frame node destroyed"); +} + + +/* + * + * Exported functions + * + */ + +struct xrt_device * +rs_hdev_create(struct xrt_prober *xp, int device_idx) +{ + struct rs_hdev *rs = U_DEVICE_ALLOCATE(struct rs_hdev, U_DEVICE_ALLOC_TRACKING_NONE, 1, 0); + rs->ll = debug_get_log_option_rs_log(); + rs->pose = (struct xrt_pose){{0, 0, 0, 1}, {0, 0, 0}}; + rs->offset = (struct xrt_pose){{0, 0, 0, 1}, {0, 0, 0}}; + + struct xrt_device *xd = &rs->xdev; + xd->name = XRT_DEVICE_REALSENSE; + xd->device_type = XRT_DEVICE_TYPE_GENERIC_TRACKER; + + snprintf(xd->str, XRT_DEVICE_NAME_LEN, "%s", DEVICE_STRING); + snprintf(xd->serial, XRT_DEVICE_NAME_LEN, "%s", DEVICE_STRING); + + snprintf(xd->tracking_origin->name, XRT_TRACKING_NAME_LEN, "%s", RS_HOST_SLAM_TRACKER_STR); + xd->tracking_origin->type = XRT_TRACKING_TYPE_EXTERNAL_SLAM; + + xd->inputs[0].name = XRT_INPUT_GENERIC_TRACKER_POSE; + + xd->orientation_tracking_supported = true; + xd->position_tracking_supported = true; + + xd->update_inputs = rs_hdev_update_inputs; + xd->get_tracked_pose = rs_hdev_get_tracked_pose; + xd->destroy = rs_hdev_destroy; + + rs_hdev_fs_create(rs, device_idx); + + bool tracked = rs_hdev_slam_track(rs, xp); + if (!tracked) { + RS_WARN("Unable to setup the SLAM tracker"); + rs_hdev_destroy(xd); + return NULL; + } + + RS_DEBUG("Host-SLAM RealSense device created"); + + return xd; +} diff --git a/src/xrt/drivers/realsense/rs_interface.h b/src/xrt/drivers/realsense/rs_interface.h index dd1d70026..0e342a5b8 100644 --- a/src/xrt/drivers/realsense/rs_interface.h +++ b/src/xrt/drivers/realsense/rs_interface.h @@ -13,7 +13,6 @@ extern "C" { #endif - /*! * @defgroup drv_rs Intel RealSense driver * @ingroup drv @@ -21,16 +20,20 @@ extern "C" { * @brief Driver for the SLAM-capable Intel Realsense devices. */ +#define RS_HOST_SLAM_TRACKER_STR "Host SLAM Tracker for RealSense" + +#define RS_TRACKING_DISABLED -1 +#define RS_TRACKING_UNSPECIFIED 0 +#define RS_TRACKING_DEVICE_SLAM 1 +#define RS_TRACKING_HOST_SLAM 2 + /*! - * Create a RelaseSense 6DOF tracker device. + * Create a auto prober for rs devices. * * @ingroup drv_rs */ -struct xrt_device * -rs_6dof_create(void); - -void -rs_update_offset(struct xrt_pose offset, struct xrt_device *xdev); +struct xrt_auto_prober * +rs_create_auto_prober(void); /*! * @dir drivers/realsense diff --git a/src/xrt/drivers/realsense/rs_prober.c b/src/xrt/drivers/realsense/rs_prober.c new file mode 100644 index 000000000..11a973cf9 --- /dev/null +++ b/src/xrt/drivers/realsense/rs_prober.c @@ -0,0 +1,252 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Realsense prober code. + * @author Christoph Haag + * @author Jakob Bornecrantz + * @ingroup drv_rs + */ + +#include +#include + +#include "xrt/xrt_config_have.h" +#include "xrt/xrt_prober.h" + +#include "util/u_misc.h" +#include "util/u_debug.h" + +#include "rs_driver.h" + +#include + +#define INFO(...) U_LOG(U_LOGGING_INFO, __VA_ARGS__); +#define WARN(...) U_LOG(U_LOGGING_WARN, __VA_ARGS__); +#define ERROR(...) U_LOG(U_LOGGING_ERROR, __VA_ARGS__); + +//! Utility for realsense API calls that can produce errors +#define DO(call, ...) \ + call(__VA_ARGS__, &e); \ + check_error(e, __FILE__, __LINE__); + +/*! + * @brief Specifies which realsense tracking to use + * -1 for DISABLED, will not create any RealSense device + * 0 for UNSPECIFIED, will decide based on what's available + * 1 for DEVICE_SLAM, will only try to use in-device SLAM tracking + * 2 for HOST_SLAM, will only try to use external SLAM tracking + */ +DEBUG_GET_ONCE_NUM_OPTION(rs_tracking, "RS_TRACKING", RS_TRACKING_UNSPECIFIED) + +static bool +check_error(rs2_error *e, const char *file, int line) +{ + if (e == NULL) { + return false; // No errors + } + + ERROR("rs_error was raised when calling %s(%s):", rs2_get_failed_function(e), rs2_get_failed_args(e)); + ERROR("%s:%d: %s", file, line, rs2_get_error_message(e)); + exit(EXIT_FAILURE); +} + +/*! + * @implements xrt_auto_prober + */ +struct rs_prober +{ + struct xrt_auto_prober base; +}; + +//! @private @memberof rs_prober +static inline struct rs_prober * +rs_prober(struct xrt_auto_prober *p) +{ + return (struct rs_prober *)p; +} + +//! @public @memberof rs_prober +static void +rs_prober_destroy(struct xrt_auto_prober *p) +{ + struct rs_prober *dp = rs_prober(p); + + free(dp); +} + +/*! + * @brief Explores a realsense device to see what SLAM capabilities it supports + * + * @param[out] out_hslam Whether it supports host-SLAM tracking (Has camera-imu streams) + * @param[out] out_dslam Whether it supports device-SLAM tracking (T26x) + */ +static void +check_slam_capabilities(rs2_device_list *device_list, int dev_idx, bool *out_hslam, bool *out_dslam) +{ + //! @todo Consider adding the sensors list to the rs_container + bool video_sensor_found = false; + bool imu_sensor_found = false; + bool pose_sensor_found = false; + + rs2_error *e = NULL; + rs2_device *device = DO(rs2_create_device, device_list, dev_idx); + rs2_sensor_list *sensors = DO(rs2_query_sensors, device); + int sensors_count = DO(rs2_get_sensors_count, sensors); + for (int i = 0; i < sensors_count; i++) { + rs2_sensor *sensor = DO(rs2_create_sensor, sensors, i); + video_sensor_found |= DO(rs2_is_sensor_extendable_to, sensor, RS2_EXTENSION_VIDEO); + imu_sensor_found |= DO(rs2_is_sensor_extendable_to, sensor, RS2_EXTENSION_MOTION_SENSOR); + pose_sensor_found |= DO(rs2_is_sensor_extendable_to, sensor, RS2_EXTENSION_POSE_SENSOR); + rs2_delete_sensor(sensor); + } + + rs2_delete_sensor_list(sensors); + rs2_delete_device(device); + + *out_hslam = video_sensor_found && imu_sensor_found; + *out_dslam = pose_sensor_found; +} + +static bool +supports_host_slam(rs2_device_list *device_list, int index) +{ + bool supported, _; + check_slam_capabilities(device_list, index, &supported, &_); + return supported; +} + +static bool +supports_device_slam(rs2_device_list *device_list, int index) +{ + bool supported, _; + check_slam_capabilities(device_list, index, &_, &supported); + return supported; +} + +//! @return index of the first device in device_list that has the requested +//! capability or -1 if none. +static int +find_capable_device(int capability, rs2_device_list *device_list) +{ + rs2_error *e = NULL; + int device_count = DO(rs2_get_device_count, device_list); + + // Determine predicate to check if a device supports the capability + bool (*supports_capability)(rs2_device_list *, int); + if (capability == RS_TRACKING_DEVICE_SLAM) { + supports_capability = supports_device_slam; + } else if (capability == RS_TRACKING_HOST_SLAM) { + supports_capability = supports_host_slam; + } else { + ERROR("Invalid capability=%d requested", capability); + return false; + } + + // Find the first device that support the capability + int cdev_idx = -1; // cdev means capable device + for (int i = 0; i < device_count; i++) { + if (supports_capability(device_list, i)) { + cdev_idx = i; + break; + } + } + + return cdev_idx; +} + +//! Implements the conditional flow to decide on how to pick which tracking to use +static struct xrt_device * +create_tracked_rs_device(struct xrt_prober *xp) +{ + rs2_error *e = NULL; + struct rs_container rsc = {0}; + int expected_tracking = debug_get_num_option_rs_tracking(); +#ifdef XRT_HAVE_SLAM + bool external_slam_supported = true; +#else + bool external_slam_supported = false; +#endif + + rsc.context = DO(rs2_create_context, RS2_API_VERSION); + rsc.device_list = DO(rs2_query_devices, rsc.context); + rsc.device_count = DO(rs2_get_device_count, rsc.device_list); + + if (rsc.device_count == 0) { + if (expected_tracking != RS_TRACKING_UNSPECIFIED) { + WARN("RS_TRACKING=%d provided but no RealSense devices found", expected_tracking); + } + rs_container_cleanup(&rsc); + return 0; + } + + int ddev_idx = find_capable_device(RS_TRACKING_DEVICE_SLAM, rsc.device_list); + bool has_ddev = ddev_idx != -1; + + int hdev_idx = find_capable_device(RS_TRACKING_HOST_SLAM, rsc.device_list); + bool has_hdev = hdev_idx != -1; + + rs_container_cleanup(&rsc); // We got ddev_idx and hdev_idx, release realsense resources + + struct xrt_device *dev = NULL; + if (expected_tracking == RS_TRACKING_HOST_SLAM) { + if (!external_slam_supported) { + ERROR("No external SLAM systems built, unable to produce host SLAM tracking"); + } else if (has_hdev) { + dev = rs_hdev_create(xp, hdev_idx); + } else { + ERROR("No RealSense devices that support external SLAM tracking were found"); + } + } else if (expected_tracking == RS_TRACKING_DEVICE_SLAM) { + if (has_ddev) { + dev = rs_ddev_create(ddev_idx); + } else { + WARN("No RealSense devices that support in-device SLAM tracking were found"); + } + } else if (expected_tracking == RS_TRACKING_UNSPECIFIED) { + if (has_ddev) { + dev = rs_ddev_create(ddev_idx); + } else if (has_hdev && external_slam_supported) { + dev = rs_hdev_create(xp, hdev_idx); + } else { + INFO("No RealSense devices that can be tracked were found"); + } + } else if (expected_tracking == RS_TRACKING_DISABLED) { + INFO("RS_TRACKING=%d (DISABLED) so skipping any RealSense device", RS_TRACKING_DISABLED); + } else { + ERROR("Invalid RS_TRACKING=%d", expected_tracking); + } + + return dev; +} + +//! @public @memberof rs_prober +static int +rs_prober_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) +{ + struct rs_prober *dp = rs_prober(xap); + (void)dp; + + struct xrt_device *dev = create_tracked_rs_device(xp); + if (!dev) { + return 0; + } + + out_xdevs[0] = dev; + return 1; +} + +struct xrt_auto_prober * +rs_create_auto_prober() +{ + struct rs_prober *dp = U_TYPED_CALLOC(struct rs_prober); + dp->base.name = "Realsense"; + dp->base.destroy = rs_prober_destroy; + dp->base.lelo_dallas_autoprobe = rs_prober_autoprobe; + + return &dp->base; +} diff --git a/src/xrt/drivers/remote/r_device.c b/src/xrt/drivers/remote/r_device.c index 4dc89c223..a765e7e76 100644 --- a/src/xrt/drivers/remote/r_device.c +++ b/src/xrt/drivers/remote/r_device.c @@ -113,8 +113,9 @@ r_device_get_tracked_pose(struct xrt_device *xdev, static void r_device_get_hand_tracking(struct xrt_device *xdev, enum xrt_input_name name, - uint64_t at_timestamp_ns, - struct xrt_hand_joint_set *out_value) + uint64_t requested_timestamp_ns, + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp_ns) { struct r_device *rd = r_device(xdev); struct r_hub *r = rd->r; @@ -136,26 +137,27 @@ r_device_get_hand_tracking(struct xrt_device *xdev, }; enum xrt_hand hand = rd->is_left ? XRT_HAND_LEFT : XRT_HAND_RIGHT; - u_hand_joints_update_curl(&rd->hand_tracking, hand, at_timestamp_ns, &values); + u_hand_joints_update_curl(&rd->hand_tracking, hand, requested_timestamp_ns, &values); - struct xrt_pose hand_on_handle_pose = { - {0, 0, 0, 1}, - {0, 0, 0}, - }; + struct xrt_pose hand_on_handle_pose = XRT_POSE_IDENTITY; struct xrt_space_relation relation; - xrt_device_get_tracked_pose(xdev, XRT_INPUT_SIMPLE_GRIP_POSE, at_timestamp_ns, &relation); + xrt_device_get_tracked_pose(xdev, XRT_INPUT_SIMPLE_GRIP_POSE, requested_timestamp_ns, &relation); u_hand_joints_set_out_data(&rd->hand_tracking, hand, &relation, &hand_on_handle_pose, out_value); + + // This is a lie + *out_timestamp_ns = requested_timestamp_ns; } static void r_device_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { // Empty + assert(false); } static void @@ -195,6 +197,7 @@ r_device_create(struct r_hub *r, bool is_left) // Print name. snprintf(rd->base.str, sizeof(rd->base.str), "Remote %s Controller", is_left ? "Left" : "Right"); + snprintf(rd->base.serial, sizeof(rd->base.str), "Remote %s Controller", is_left ? "Left" : "Right"); // Inputs and outputs. rd->base.inputs[0].name = XRT_INPUT_SIMPLE_SELECT_CLICK; diff --git a/src/xrt/drivers/remote/r_hmd.c b/src/xrt/drivers/remote/r_hmd.c index cc5202a83..bfefd822f 100644 --- a/src/xrt/drivers/remote/r_hmd.c +++ b/src/xrt/drivers/remote/r_hmd.c @@ -77,7 +77,8 @@ static void r_hmd_get_hand_tracking(struct xrt_device *xdev, enum xrt_input_name name, uint64_t at_timestamp_ns, - struct xrt_hand_joint_set *out_value) + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp_ns) { struct r_hmd *rh = r_hmd(xdev); (void)rh; @@ -85,29 +86,12 @@ r_hmd_get_hand_tracking(struct xrt_device *xdev, static void r_hmd_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { - struct xrt_pose pose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; - bool adjust = view_index == 0; - - pose.position.x = eye_relation->x / 2.0f; - pose.position.y = eye_relation->y / 2.0f; - pose.position.z = eye_relation->z / 2.0f; - - // Adjust for left/right while also making sure there aren't any -0.f. - if (pose.position.x > 0.0f && adjust) { - pose.position.x = -pose.position.x; - } - if (pose.position.y > 0.0f && adjust) { - pose.position.y = -pose.position.y; - } - if (pose.position.z > 0.0f && adjust) { - pose.position.z = -pose.position.z; - } - - *out_pose = pose; + (void)xdev; + u_device_get_view_pose(eye_relation, view_index, out_pose); } static void @@ -148,6 +132,7 @@ r_hmd_create(struct r_hub *r) // Print name. snprintf(rh->base.str, sizeof(rh->base.str), "Remote HMD"); + snprintf(rh->base.serial, sizeof(rh->base.serial), "Remote HMD"); // Setup info. struct u_device_simple_info info; diff --git a/src/xrt/drivers/remote/r_hub.c b/src/xrt/drivers/remote/r_hub.c index ab0517c90..35ec0cfb3 100644 --- a/src/xrt/drivers/remote/r_hub.c +++ b/src/xrt/drivers/remote/r_hub.c @@ -28,9 +28,13 @@ #ifndef __USE_MISC #define __USE_MISC // SOL_TCP on C11 #endif +#ifndef _BSD_SOURCE +#define _BSD_SOURCE // same, but for musl +#endif #include + /* * * Function. diff --git a/src/xrt/drivers/survive/survive_driver.c b/src/xrt/drivers/survive/survive_driver.c index b90d3abb5..0b5a314b0 100644 --- a/src/xrt/drivers/survive/survive_driver.c +++ b/src/xrt/drivers/survive/survive_driver.c @@ -1,10 +1,11 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: Apache-2.0 /*! * @file * @brief Adapter to Libsurvive. * @author Christoph Haag * @author Jakob Bornecrantz + * @author Moses Turner * @ingroup drv_survive */ @@ -17,6 +18,7 @@ #include #include "math/m_api.h" +#include "tracking/t_tracking.h" #include "xrt/xrt_device.h" #include "util/u_debug.h" #include "util/u_device.h" @@ -24,33 +26,46 @@ #include "util/u_time.h" #include "util/u_device.h" #include "util/u_distortion_mesh.h" +#include "util/u_config_json.h" -#include "../auxiliary/os/os_time.h" +#include "os/os_threading.h" + +#include "os/os_time.h" #include "xrt/xrt_prober.h" #include "survive_interface.h" #include "survive_api.h" -#include "survive_wrap.h" - #include "util/u_json.h" #include "util/u_hand_tracking.h" #include "util/u_logging.h" +#include "math/m_relation_history.h" #include "math/m_predict.h" -// typically HMD config is available at around 2 seconds after init -#define WAIT_TIMEOUT 5.0 +#include "vive/vive_config.h" -// public documentation -//! @todo move to vive_protocol -#define INDEX_MIN_IPD 0.058 -#define INDEX_MAX_IPD 0.07 +#include "../ht/ht_interface.h" +#include "../multi_wrapper/multi.h" +#include "xrt/xrt_config_drivers.h" -#define DEFAULT_HAPTIC_FREQ 150.0f -#define MIN_HAPTIC_DURATION 0.05f +#include "survive_driver.h" + +// If we haven't gotten a config for devices this long after startup, just start without those devices +#define DEFAULT_WAIT_TIMEOUT 3.5f + +// index in sys->controllers[] array +#define SURVIVE_LEFT_CONTROLLER_INDEX 0 +#define SURVIVE_RIGHT_CONTROLLER_INDEX 1 +#define SURVIVE_NON_CONTROLLER_START 2 + +//! excl HMD we support 16 devices (controllers, trackers, ...) +#define MAX_TRACKED_DEVICE_COUNT 16 + +// initializing survive_driver once creates xrt_devices for all connected devices +static bool survive_already_initialized = false; #define SURVIVE_TRACE(d, ...) U_LOG_XDEV_IFL_T(&d->base, d->sys->ll, __VA_ARGS__) #define SURVIVE_DEBUG(d, ...) U_LOG_XDEV_IFL_D(&d->base, d->sys->ll, __VA_ARGS__) @@ -92,30 +107,15 @@ enum input_index VIVE_CONTROLLER_HAND_TRACKING, + VIVE_TRACKER_POSE, + VIVE_CONTROLLER_MAX_INDEX, }; -// also used as index in sys->controllers[] array -typedef enum +enum DeviceType { - SURVIVE_LEFT_CONTROLLER = 0, - SURVIVE_RIGHT_CONTROLLER = 1, - SURVIVE_HMD = 2, -} SurviveDeviceType; - -static bool survive_already_initialized = false; - -enum VIVE_VARIANT -{ - VIVE_UNKNOWN = 0, - VIVE_VARIANT_VIVE, - VIVE_VARIANT_PRO, - VIVE_VARIANT_VALVE_INDEX, - VIVE_VARIANT_HTC_VIVE_CONTROLLER, - VIVE_VARIANT_VALVE_INDEX_LEFT_CONTROLLER, - VIVE_VARIANT_VALVE_INDEX_RIGHT_CONTROLLER, - VIVE_VARIANT_TRACKER_V1, - VIVE_VARIANT_TRACKER_v2 + DEVICE_TYPE_HMD, + DEVICE_TYPE_CONTROLLER }; /*! @@ -127,18 +127,22 @@ struct survive_device struct survive_system *sys; const SurviveSimpleObject *survive_obj; - struct xrt_space_relation last_relation; + struct m_relation_history *relation_hist; - int num; + //! Number of inputs. + size_t num_last_inputs; + //! Array of input structs. + struct xrt_input *last_inputs; - enum VIVE_VARIANT variant; + enum DeviceType device_type; union { struct { float proximity; // [0,1] float ipd; - struct xrt_quat rot[2]; + + struct vive_config config; } hmd; struct @@ -146,14 +150,12 @@ struct survive_device float curl[XRT_FINGER_COUNT]; uint64_t curl_ts[XRT_FINGER_COUNT]; struct u_hand_tracking hand_tracking; + + struct vive_controller_config config; } ctrl; }; - struct u_vive_values distortion[2]; }; -//! @todo support more devices (trackers, ...) -#define CONTROLLER_COUNT 2 - /*! * @extends xrt_tracking_origin */ @@ -162,32 +164,59 @@ struct survive_system struct xrt_tracking_origin base; SurviveSimpleContext *ctx; struct survive_device *hmd; - struct survive_device *controllers[CONTROLLER_COUNT]; + struct survive_device *controllers[MAX_TRACKED_DEVICE_COUNT]; enum u_logging_level ll; + + float wait_timeout; + + struct os_thread_helper event_thread; + struct os_mutex lock; }; static void survive_device_destroy(struct xrt_device *xdev) { + if (!xdev) { + return; + } + U_LOG_D("destroying survive device"); struct survive_device *survive = (struct survive_device *)xdev; - if (survive == survive->sys->hmd) + if (survive == survive->sys->hmd) { + vive_config_teardown(&survive->hmd.config); survive->sys->hmd = NULL; - if (survive == survive->sys->controllers[SURVIVE_LEFT_CONTROLLER]) - survive->sys->controllers[SURVIVE_LEFT_CONTROLLER] = NULL; - if (survive == survive->sys->controllers[SURVIVE_RIGHT_CONTROLLER]) - survive->sys->controllers[SURVIVE_RIGHT_CONTROLLER] = NULL; - - if (survive->sys->hmd == NULL && survive->sys->controllers[SURVIVE_LEFT_CONTROLLER] == NULL && - survive->sys->controllers[SURVIVE_RIGHT_CONTROLLER] == NULL) { - U_LOG_D("Tearing down libsurvive context"); - survive_simple_close(survive->sys->ctx); - - free(survive->sys); + } + for (int i = 0; i < MAX_TRACKED_DEVICE_COUNT; i++) { + if (survive == survive->sys->controllers[i]) { + survive->sys->controllers[i] = NULL; + } } - free(survive); + bool all_null = true; + for (int i = 0; i < MAX_TRACKED_DEVICE_COUNT; i++) { + if (survive->sys->controllers[i] != 0) { + all_null = false; + } + } + + if (survive->sys->hmd == NULL && all_null) { + U_LOG_D("Tearing down libsurvive context"); + os_thread_helper_stop(&survive->sys->event_thread); + os_thread_helper_destroy(&survive->sys->event_thread); + + // Now that the thread is not running we can destroy the lock. + os_mutex_destroy(&survive->sys->lock); + + U_LOG_D("Stopped libsurvive event thread"); + + survive_simple_close(survive->sys->ctx); + free(survive->sys); + } + m_relation_history_destroy(&survive->relation_hist); + + free(survive->last_inputs); + u_device_free(&survive->base); } // libsurvive timecode may not be exactly comparable with monotonic ns. @@ -200,50 +229,24 @@ survive_timecode_now_s() return ((double)tv.tv_usec) / 1000000. + (tv.tv_sec); } -static void -_get_survive_pose(struct survive_device *survive, uint64_t at_timestamp_ns, struct xrt_space_relation *out_relation) +static timepoint_ns +survive_timecode_to_monotonic(double timecode) { - const SurviveSimpleObject *survive_object = survive->survive_obj; - - out_relation->relation_flags = XRT_SPACE_RELATION_BITMASK_NONE; - - if (survive_simple_object_get_type(survive_object) != SurviveSimpleObject_OBJECT && - survive_simple_object_get_type(survive_object) != SurviveSimpleObject_HMD) { - return; - } - - // Initially pose can be zeroed. That's okay, we report it without - // orientation etc. valid flag then. - SurvivePose pose; - SurviveVelocity vel; - - // "device time" in seconds - double timecode_s = survive_simple_object_get_latest_pose(survive_object, &pose); - - double vel_timecode_s = survive_simple_object_get_latest_velocity(survive_object, &vel); - (void)vel_timecode_s; - - // do calculations in ns due to large numbers - timepoint_ns timecode_ns = time_s_to_ns(timecode_s); - - timepoint_ns monotonic_now_ns = os_monotonic_get_ns(); - timepoint_ns remaining_ns = at_timestamp_ns - monotonic_now_ns; - + timepoint_ns timecode_ns = time_s_to_ns(timecode); timepoint_ns survive_now_ns = time_s_to_ns(survive_timecode_now_s()); - timepoint_ns survive_pose_age_ns = survive_now_ns - timecode_ns; - timepoint_ns prediction_ns = remaining_ns + survive_pose_age_ns; + timepoint_ns timecode_age_ns = survive_now_ns - timecode_ns; - double prediction_s = time_ns_to_s(prediction_ns); + timepoint_ns now = os_monotonic_get_ns(); + timepoint_ns timestamp = now - timecode_age_ns; - SURVIVE_TRACE(survive, - "dev %s At %ldns: Pose requested for +%ldns (%ldns). " - "Libsurvive Pose Timecode: %ldns, Pose gotten at -%ldns " - "from now (%ldns), predicting %ldns", - survive->base.str, monotonic_now_ns, remaining_ns, at_timestamp_ns, timecode_ns, - survive_pose_age_ns, survive_now_ns, prediction_ns); + return timestamp; +} - struct xrt_quat out_rot = {.x = pose.Rot[1], .y = pose.Rot[2], .z = pose.Rot[3], .w = pose.Rot[0]}; +static void +pose_to_relation(const SurvivePose *pose, const SurviveVelocity *vel, struct xrt_space_relation *out_relation) +{ + struct xrt_quat out_rot = {.x = pose->Rot[1], .y = pose->Rot[2], .z = pose->Rot[3], .w = pose->Rot[0]}; /* libsurvive looks down when it should be looking forward, so * rotate the quat. @@ -261,18 +264,17 @@ _get_survive_pose(struct survive_device *survive, uint64_t at_timestamp_ns, stru // just to be sure math_quat_normalize(&out_rot); - - out_relation->pose.orientation = out_rot; /* switch -y, z axes to go from libsurvive coordinate system to ours */ - out_relation->pose.position.x = pose.Pos[0]; - out_relation->pose.position.y = pose.Pos[2]; - out_relation->pose.position.z = -pose.Pos[1]; + out_relation->pose.position.x = pose->Pos[0]; + out_relation->pose.position.y = pose->Pos[2]; + out_relation->pose.position.z = -pose->Pos[1]; - struct xrt_vec3 linear_vel = {.x = vel.Pos[0], .y = vel.Pos[2], .z = -vel.Pos[1]}; + struct xrt_vec3 linear_vel = {.x = vel->Pos[0], .y = vel->Pos[2], .z = -vel->Pos[1]}; - struct xrt_vec3 angular_vel = {.x = vel.AxisAngleRot[0], .y = vel.AxisAngleRot[2], .z = -vel.AxisAngleRot[1]}; + struct xrt_vec3 angular_vel = { + .x = vel->AxisAngleRot[0], .y = vel->AxisAngleRot[2], .z = -vel->AxisAngleRot[1]}; if (math_quat_validate(&out_rot)) { out_relation->relation_flags |= @@ -296,57 +298,21 @@ _get_survive_pose(struct survive_device *survive, uint64_t at_timestamp_ns, stru out_relation->relation_flags |= XRT_SPACE_RELATION_ANGULAR_VELOCITY_VALID_BIT; } } - - SURVIVE_TRACE(survive, "Predicting %fs for %s", prediction_s, survive->base.str); - - struct xrt_space_relation rel = *out_relation; - m_predict_relation(&rel, prediction_s, out_relation); } -//! @todo: Hotplugging, with get_variant_from_json() -#if 0 static bool -_update_survive_devices(struct survive_system *sys) +verify_device_name(struct survive_device *survive, enum xrt_input_name name) { - //! @todo better method - if (sys->hmd->survive_obj && sys->controllers[0]->survive_obj && - sys->controllers[1]->survive_obj) - return true; - - SurviveSimpleContext *ctx = sys->ctx; - - for (const SurviveSimpleObject *it = - survive_simple_get_first_object(ctx); - it != 0; it = survive_simple_get_next_object(ctx, it)) { - const char *codename = survive_simple_object_name(it); - - enum SurviveSimpleObject_type type = - survive_simple_object_get_type(it); - if (type == SurviveSimpleObject_HMD && - sys->hmd->survive_obj == NULL) { - U_LOG_D("Found HMD: %s", codename); - sys->hmd->survive_obj = it; - } - if (type == SurviveSimpleObject_OBJECT) { - for (int i = 0; i < CONTROLLER_COUNT; i++) { - if (sys->controllers[i]->survive_obj == it) { - break; - } - - if (sys->controllers[i]->survive_obj == NULL) { - U_LOG_D("Found Controller %d: %s", i, - codename); - sys->controllers[i]->survive_obj = it; - break; - } - } - } - } - - return true; + switch (survive->device_type) { + case DEVICE_TYPE_HMD: return name == XRT_INPUT_GENERIC_HEAD_POSE; + case DEVICE_TYPE_CONTROLLER: + return name == XRT_INPUT_INDEX_AIM_POSE || name == XRT_INPUT_INDEX_GRIP_POSE || + name == XRT_INPUT_VIVE_AIM_POSE || name == XRT_INPUT_VIVE_GRIP_POSE || + name == XRT_INPUT_GENERIC_TRACKER_POSE; + }; + return false; } -#endif static void survive_device_get_tracked_pose(struct xrt_device *xdev, @@ -355,25 +321,17 @@ survive_device_get_tracked_pose(struct xrt_device *xdev, struct xrt_space_relation *out_relation) { struct survive_device *survive = (struct survive_device *)xdev; - if ((survive == survive->sys->hmd && name != XRT_INPUT_GENERIC_HEAD_POSE) || - ((survive == survive->sys->controllers[0] || survive == survive->sys->controllers[1]) && - (name != XRT_INPUT_INDEX_AIM_POSE && name != XRT_INPUT_INDEX_GRIP_POSE) && - (name != XRT_INPUT_VIVE_AIM_POSE && name != XRT_INPUT_VIVE_GRIP_POSE))) { - + if (!verify_device_name(survive, name)) { SURVIVE_ERROR(survive, "unknown input name"); return; } - //_update_survive_devices(survive->sys); if (!survive->survive_obj) { // U_LOG_D("Obj not set for %p", (void*)survive); return; } - - _get_survive_pose(survive, at_timestamp_ns, out_relation); - - survive->last_relation = *out_relation; + m_relation_history_get(survive->relation_hist, out_relation, at_timestamp_ns); struct xrt_pose *p = &out_relation->pose; SURVIVE_TRACE(survive, "GET_POSITION (%f %f %f) GET_ORIENTATION (%f, %f, %f, %f)", p->position.x, p->position.y, @@ -430,14 +388,12 @@ survive_controller_device_set_output(struct xrt_device *xdev, enum xrt_output_na } } -#define PI 3.14159265358979323846 -#define DEG_TO_RAD(DEG) (DEG * PI / 180.) - static void survive_controller_get_hand_tracking(struct xrt_device *xdev, enum xrt_input_name name, uint64_t at_timestamp_ns, - struct xrt_hand_joint_set *out_value) + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp_ns) { struct survive_device *survive = (struct survive_device *)xdev; @@ -447,15 +403,15 @@ survive_controller_get_hand_tracking(struct xrt_device *xdev, } - bool left = survive->variant == VIVE_VARIANT_VALVE_INDEX_LEFT_CONTROLLER; + bool left = survive->ctrl.config.variant == CONTROLLER_INDEX_LEFT; enum xrt_hand hand = left ? XRT_HAND_LEFT : XRT_HAND_RIGHT; float thumb_curl = 0.0f; //! @todo place thumb preciely on the button that is touched/pressed - if (survive->base.inputs[VIVE_CONTROLLER_A_TOUCH].value.boolean || - survive->base.inputs[VIVE_CONTROLLER_B_TOUCH].value.boolean || - survive->base.inputs[VIVE_CONTROLLER_THUMBSTICK_TOUCH].value.boolean || - survive->base.inputs[VIVE_CONTROLLER_TRACKPAD_TOUCH].value.boolean) { + if (survive->last_inputs[VIVE_CONTROLLER_A_TOUCH].value.boolean || + survive->last_inputs[VIVE_CONTROLLER_B_TOUCH].value.boolean || + survive->last_inputs[VIVE_CONTROLLER_THUMBSTICK_TOUCH].value.boolean || + survive->last_inputs[VIVE_CONTROLLER_TRACKPAD_TOUCH].value.boolean) { thumb_curl = 1.0; } @@ -475,37 +431,34 @@ survive_controller_get_hand_tracking(struct xrt_device *xdev, struct xrt_pose hand_on_handle_pose; u_hand_joints_offset_valve_index_controller(hand, &static_offset, &hand_on_handle_pose); - u_hand_joints_set_out_data(&survive->ctrl.hand_tracking, hand, &survive->last_relation, &hand_on_handle_pose, - out_value); + struct xrt_space_relation hand_relation; + + m_relation_history_get(survive->relation_hist, &hand_relation, at_timestamp_ns); + + u_hand_joints_set_out_data(&survive->ctrl.hand_tracking, hand, &hand_relation, &hand_on_handle_pose, out_value); + + // This is the truth - we pose-predicted or interpolated all the way up to `at_timestamp_ns`. + *out_timestamp_ns = at_timestamp_ns; + + // This is a lie - apparently libsurvive doesn't report controller tracked/untracked state, so just say that the + // hand is being tracked + out_value->is_active = true; } static void survive_device_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { - struct xrt_pose pose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; - bool adjust = view_index == 0; + // Only supports two views. + assert(view_index < 2); + u_device_get_view_pose(eye_relation, view_index, out_pose); + + // This is for the Index' canted displays, on the Vive [Pro] they are identity. struct survive_device *survive = (struct survive_device *)xdev; - pose.orientation = survive->hmd.rot[view_index]; - pose.position.x = eye_relation->x / 2.0f; - pose.position.y = eye_relation->y / 2.0f; - pose.position.z = eye_relation->z / 2.0f; - - // Adjust for left/right while also making sure there aren't any -0.f. - if (pose.position.x > 0.0f && adjust) { - pose.position.x = -pose.position.x; - } - if (pose.position.y > 0.0f && adjust) { - pose.position.y = -pose.position.y; - } - if (pose.position.z > 0.0f && adjust) { - pose.position.z = -pose.position.z; - } - - *out_pose = pose; + out_pose->orientation = survive->hmd.config.display.rot[view_index]; } enum InputComponent @@ -521,7 +474,7 @@ struct Axis enum InputComponent comp; }; -struct Axis axes[255] = { +static struct Axis axes[255] = { [SURVIVE_AXIS_TRIGGER] = { .input = VIVE_CONTROLLER_TRIGGER_VALUE, @@ -566,7 +519,7 @@ update_axis(struct survive_device *survive, struct Axis *axis, const SurviveSimp return false; } - struct xrt_input *in = &survive->base.inputs[axis->input]; + struct xrt_input *in = &survive->last_inputs[axis->input]; float fval = e->axis_val[i]; @@ -612,7 +565,7 @@ struct Button buttons[255] = { }; static bool -update_button(struct survive_device *survive, const struct SurviveSimpleButtonEvent *e, uint64_t now) +update_button(struct survive_device *survive, const struct SurviveSimpleButtonEvent *e, timepoint_ns ts) { if (e->event_type == SURVIVE_INPUT_EVENT_NONE) { return true; @@ -624,24 +577,24 @@ update_button(struct survive_device *survive, const struct SurviveSimpleButtonEv if (e_type == SURVIVE_INPUT_EVENT_BUTTON_UP) { enum input_index index = buttons[btn_id].click; - struct xrt_input *input = &survive->base.inputs[index]; + struct xrt_input *input = &survive->last_inputs[index]; input->value.boolean = false; - input->timestamp = now; + input->timestamp = ts; } else if (e_type == SURVIVE_INPUT_EVENT_BUTTON_DOWN) { enum input_index index = buttons[btn_id].click; - struct xrt_input *input = &survive->base.inputs[index]; + struct xrt_input *input = &survive->last_inputs[index]; input->value.boolean = true; - input->timestamp = now; + input->timestamp = ts; } else if (e_type == SURVIVE_INPUT_EVENT_TOUCH_UP) { enum input_index index = buttons[btn_id].touch; - struct xrt_input *input = &survive->base.inputs[index]; + struct xrt_input *input = &survive->last_inputs[index]; input->value.boolean = false; - input->timestamp = now; + input->timestamp = ts; } else if (e_type == SURVIVE_INPUT_EVENT_TOUCH_DOWN) { enum input_index index = buttons[btn_id].touch; - struct xrt_input *input = &survive->base.inputs[index]; + struct xrt_input *input = &survive->last_inputs[index]; input->value.boolean = true; - input->timestamp = now; + input->timestamp = ts; } return true; @@ -659,47 +612,49 @@ _calculate_squeeze_value(struct survive_device *survive) } static void -_process_button_event(struct survive_device *survive, const struct SurviveSimpleButtonEvent *e, int64_t now) +_process_button_event(struct survive_device *survive, const struct SurviveSimpleButtonEvent *e) { + timepoint_ns ts = survive_timecode_to_monotonic(e->time); + ; if (e->event_type == SURVIVE_INPUT_EVENT_AXIS_CHANGED) { for (int i = 0; i < e->axis_count; i++) { struct Axis *axis = &axes[e->axis_ids[i]]; float val = e->axis_val[i]; - if (update_axis(survive, axis, e, i, now)) { + if (update_axis(survive, axis, e, i, ts)) { } else if (e->axis_ids[i] == SURVIVE_AXIS_TRIGGER_FINGER_PROXIMITY) { survive->ctrl.curl[XRT_FINGER_INDEX] = val; - survive->ctrl.curl_ts[XRT_FINGER_INDEX] = now; + survive->ctrl.curl_ts[XRT_FINGER_INDEX] = ts; } else if (e->axis_ids[i] == SURVIVE_AXIS_MIDDLE_FINGER_PROXIMITY) { survive->ctrl.curl[XRT_FINGER_MIDDLE] = val; - survive->ctrl.curl_ts[XRT_FINGER_MIDDLE] = now; + survive->ctrl.curl_ts[XRT_FINGER_MIDDLE] = ts; } else if (e->axis_ids[i] == SURVIVE_AXIS_RING_FINGER_PROXIMITY) { survive->ctrl.curl[XRT_FINGER_RING] = val; - survive->ctrl.curl_ts[XRT_FINGER_RING] = now; + survive->ctrl.curl_ts[XRT_FINGER_RING] = ts; } else if (e->axis_ids[i] == SURVIVE_AXIS_PINKY_FINGER_PROXIMITY) { survive->ctrl.curl[XRT_FINGER_LITTLE] = val; - survive->ctrl.curl_ts[XRT_FINGER_LITTLE] = now; + survive->ctrl.curl_ts[XRT_FINGER_LITTLE] = ts; } else { SURVIVE_DEBUG(survive, "axis id: %d val %f", e->axis_ids[i], e->axis_val[i]); } } - struct xrt_input *squeeze_value_in = &survive->base.inputs[VIVE_CONTROLLER_SQUEEZE_VALUE]; + struct xrt_input *squeeze_value_in = &survive->last_inputs[VIVE_CONTROLLER_SQUEEZE_VALUE]; float prev_squeeze_value = squeeze_value_in->value.vec1.x; float squeeze_value = _calculate_squeeze_value(survive); if (prev_squeeze_value != squeeze_value) { squeeze_value_in->value.vec1.x = squeeze_value; - squeeze_value_in->timestamp = now; + squeeze_value_in->timestamp = ts; } } - update_button(survive, e, now); + update_button(survive, e, ts); } static void -_process_hmd_button_event(struct survive_device *survive, const struct SurviveSimpleButtonEvent *e, int64_t now) +_process_hmd_button_event(struct survive_device *survive, const struct SurviveSimpleButtonEvent *e) { if (e->event_type == SURVIVE_INPUT_EVENT_AXIS_CHANGED) { for (int i = 0; i < e->axis_count; i++) { @@ -747,11 +702,11 @@ _process_hmd_button_event(struct survive_device *survive, const struct SurviveSi static struct survive_device * get_device_by_object(struct survive_system *sys, const SurviveSimpleObject *object) { - if (sys->hmd->survive_obj == object) { + if (sys->hmd != NULL && sys->hmd->survive_obj == object) { return sys->hmd; } - for (int i = 0; i < CONTROLLER_COUNT; i++) { + for (int i = 0; i < MAX_TRACKED_DEVICE_COUNT; i++) { if (sys->controllers[i] == NULL) { continue; } @@ -764,214 +719,113 @@ get_device_by_object(struct survive_system *sys, const SurviveSimpleObject *obje } static void -_process_event(struct survive_device *survive, struct SurviveSimpleEvent *event, int64_t now) +add_device(struct survive_system *ss, const struct SurviveSimpleConfigEvent *e); + +static void +_process_pose_event(struct survive_device *survive, const struct SurviveSimplePoseUpdatedEvent *e) +{ + struct xrt_space_relation rel; + timepoint_ns ts; + pose_to_relation(&e->pose, &e->velocity, &rel); + ts = survive_timecode_to_monotonic(e->time); + m_relation_history_push(survive->relation_hist, &rel, ts); + + SURVIVE_TRACE(survive, "Process pose event for %s", survive->base.str); +} + +static void +_process_event(struct survive_system *ss, struct SurviveSimpleEvent *event) { switch (event->event_type) { case SurviveSimpleEventType_ButtonEvent: { const struct SurviveSimpleButtonEvent *e = survive_simple_get_button_event(event); - struct survive_device *event_device = survive; - - if (e->object != survive->survive_obj) { - event_device = get_device_by_object(survive->sys, e->object); - } - + struct survive_device *event_device = get_device_by_object(ss, e->object); if (event_device == NULL) { - SURVIVE_ERROR(survive, "Event for unknown object not handled"); + U_LOG_IFL_I(ss->ll, "Event for unknown object not handled"); return; } // hmd & controller axes have overlapping enum indices - if (event_device == survive->sys->hmd) { - _process_hmd_button_event(event_device, e, now); + if (event_device == ss->hmd) { + _process_hmd_button_event(event_device, e); } else { - _process_button_event(event_device, e, now); + _process_button_event(event_device, e); } break; } + case SurviveSimpleEventType_ConfigEvent: { + const struct SurviveSimpleConfigEvent *e = survive_simple_get_config_event(event); + enum SurviveSimpleObject_type t = survive_simple_object_get_type(e->object); + const char *name = survive_simple_object_name(e->object); + U_LOG_IFL_D(ss->ll, "Processing config for object name %s: type %d", name, t); + add_device(ss, e); + break; + } + case SurviveSimpleEventType_PoseUpdateEvent: { + const struct SurviveSimplePoseUpdatedEvent *e = survive_simple_get_pose_updated_event(event); + + struct survive_device *event_device = get_device_by_object(ss, e->object); + if (event_device == NULL) { + U_LOG_IFL_E(ss->ll, "Event for unknown object not handled"); + return; + } + + _process_pose_event(event_device, e); + break; + } + case SurviveSimpleEventType_DeviceAdded: { + U_LOG_IFL_W(ss->ll, "Device added event, but hotplugging not implemented yet"); + break; + } case SurviveSimpleEventType_None: break; - default: SURVIVE_ERROR(survive, "Unknown event %d", event->event_type); + default: U_LOG_IFL_E(ss->ll, "Unknown event %d", event->event_type); } } - static void survive_device_update_inputs(struct xrt_device *xdev) { struct survive_device *survive = (struct survive_device *)xdev; - uint64_t now = os_monotonic_get_ns(); + os_mutex_lock(&survive->sys->lock); - /* one event queue for all devices. _process_events() updates all - devices, not just this survive device. */ - - struct SurviveSimpleEvent event = {0}; - while (survive_simple_next_event(survive->sys->ctx, &event) != SurviveSimpleEventType_None) { - _process_event(survive, &event, now); - } -} - -static bool -wait_for_device_config(const struct SurviveSimpleObject *sso) -{ - // if not backed by a survive object, we will never get a config - if (!survive_has_obj(sso)) { - return false; + for (size_t i = 0; i < survive->base.num_inputs; i++) { + survive->base.inputs[i] = survive->last_inputs[i]; } - double start = time_ns_to_s(os_monotonic_get_ns()); - do { - if (survive_config_ready(sso)) { - return true; - } - os_nanosleep(1000 * 1000 * 100); - } while (time_ns_to_s(os_monotonic_get_ns()) - start < WAIT_TIMEOUT); - - return false; -} - -void -print_vec3(const char *title, struct xrt_vec3 *vec) -{ - U_LOG_D("%s = %f %f %f", title, (double)vec->x, (double)vec->y, (double)vec->z); -} - -static long long -_json_to_int(const cJSON *item) -{ - if (item != NULL) { - return item->valueint; - } else { - return 0; - } -} - -static bool -_json_get_matrix_3x3(const cJSON *json, const char *name, struct xrt_matrix_3x3 *result) -{ - const cJSON *vec3_arr = cJSON_GetObjectItemCaseSensitive(json, name); - - // Some sanity checking. - if (vec3_arr == NULL || cJSON_GetArraySize(vec3_arr) != 3) { - return false; - } - - size_t total = 0; - const cJSON *vec = NULL; - cJSON_ArrayForEach(vec, vec3_arr) - { - assert(cJSON_GetArraySize(vec) == 3); - const cJSON *elem = NULL; - cJSON_ArrayForEach(elem, vec) - { - // Just in case. - if (total >= 9) { - break; - } - - assert(cJSON_IsNumber(elem)); - result->v[total++] = (float)elem->valuedouble; - } - } - - return true; -} - -static float -_json_get_float(const cJSON *json, const char *name) -{ - const cJSON *item = cJSON_GetObjectItemCaseSensitive(json, name); - return (float)item->valuedouble; -} - -static long long -_json_get_int(const cJSON *json, const char *name) -{ - const cJSON *item = cJSON_GetObjectItemCaseSensitive(json, name); - return _json_to_int(item); -} - -static void -_get_color_coeffs(struct u_vive_values *values, const cJSON *coeffs, uint8_t eye, uint8_t channel) -{ - // For Vive this is 8 with only 3 populated. - // For Index this is 4 with all values populated. - const cJSON *item = NULL; - size_t i = 0; - cJSON_ArrayForEach(item, coeffs) - { - values->coefficients[channel][i] = (float)item->valuedouble; - ++i; - if (i == 4) { - break; - } - } -} - -static void -get_distortion_properties(struct survive_device *d, const cJSON *eye_transform_json, uint8_t eye) -{ - const cJSON *eye_json = cJSON_GetArrayItem(eye_transform_json, eye); - if (eye_json == NULL) { - return; - } - - struct xrt_matrix_3x3 rot = {0}; - if (_json_get_matrix_3x3(eye_json, "eye_to_head", &rot)) { - math_quat_from_matrix_3x3(&rot, &d->hmd.rot[eye]); - } - - // TODO: store grow_for_undistort per eye - // clang-format off - d->distortion[eye].grow_for_undistort = _json_get_float(eye_json, "grow_for_undistort"); - d->distortion[eye].undistort_r2_cutoff = _json_get_float(eye_json, "undistort_r2_cutoff"); - // clang-format on - - const char *names[3] = { - "distortion_red", - "distortion", - "distortion_blue", - }; - - for (int i = 0; i < 3; i++) { - const cJSON *distortion = cJSON_GetObjectItemCaseSensitive(eye_json, names[i]); - if (distortion == NULL) { - continue; - } - - d->distortion[eye].center[i].x = _json_get_float(distortion, "center_x"); - d->distortion[eye].center[i].y = _json_get_float(distortion, "center_y"); - - const cJSON *coeffs = cJSON_GetObjectItemCaseSensitive(distortion, "coeffs"); - if (coeffs != NULL) { - _get_color_coeffs(&d->distortion[eye], coeffs, eye, i); - } - } + os_mutex_unlock(&survive->sys->lock); } static bool compute_distortion(struct xrt_device *xdev, int view, float u, float v, struct xrt_uv_triplet *result) { struct survive_device *d = (struct survive_device *)xdev; - return u_compute_distortion_vive(&d->distortion[view], u, v, result); + return u_compute_distortion_vive(&d->hmd.config.distortion[view], u, v, result); } static bool -_create_hmd_device(struct survive_system *sys, enum VIVE_VARIANT variant, const SurviveSimpleObject *sso) +_create_hmd_device(struct survive_system *sys, const struct SurviveSimpleObject *sso, char *conf_str) { + enum u_device_alloc_flags flags = (enum u_device_alloc_flags)U_DEVICE_ALLOC_HMD; int inputs = 1; int outputs = 0; struct survive_device *survive = U_DEVICE_ALLOCATE(struct survive_device, flags, inputs, outputs); + + if (!vive_config_parse(&survive->hmd.config, conf_str, sys->ll)) { + free(survive); + return false; + } + sys->hmd = survive; survive->sys = sys; survive->survive_obj = sso; - survive->variant = variant; + survive->device_type = DEVICE_TYPE_HMD; survive->base.name = XRT_DEVICE_GENERIC_HMD; - snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "Survive HMD"); survive->base.destroy = survive_device_destroy; survive->base.update_inputs = survive_device_update_inputs; survive->base.get_tracked_pose = survive_device_get_tracked_pose; @@ -979,82 +833,48 @@ _create_hmd_device(struct survive_system *sys, enum VIVE_VARIANT variant, const survive->base.tracking_origin = &sys->base; SURVIVE_INFO(survive, "survive HMD present"); + m_relation_history_create(&survive->relation_hist); - survive->base.hmd->blend_mode = XRT_BLEND_MODE_OPAQUE; - char *json_string = survive_get_json_config(survive->survive_obj); - cJSON *json = cJSON_Parse(json_string); - if (!cJSON_IsObject(json)) { - SURVIVE_ERROR(survive, "Could not parse JSON data."); - return false; + size_t idx = 0; + survive->base.hmd->blend_modes[idx++] = XRT_BLEND_MODE_OPAQUE; + survive->base.hmd->num_blend_modes = idx; + + switch (survive->hmd.config.variant) { + case VIVE_VARIANT_VIVE: snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "HTC Vive (libsurvive)"); break; + case VIVE_VARIANT_PRO: snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "HTC Vive Pro (libsurvive)"); break; + case VIVE_VARIANT_INDEX: snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "Valve Index (libsurvive)"); break; + case VIVE_UNKNOWN: snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "Unknown HMD (libsurvive)"); break; } - + snprintf(survive->base.serial, XRT_DEVICE_NAME_LEN, "%s", survive->hmd.config.firmware.device_serial_number); // TODO: Replace hard coded values from OpenHMD with config double w_meters = 0.122822 / 2.0; double h_meters = 0.068234; double lens_horizontal_separation = 0.057863; double eye_to_screen_distance = 0.023226876441867737; - if (survive->variant == VIVE_VARIANT_VALVE_INDEX) { - lens_horizontal_separation = 0.06; - h_meters = 0.07; - // eye relief knob adjusts this around [0.0255(near)-0.275(far)] - eye_to_screen_distance = 0.0255; - } + uint32_t w_pixels = survive->hmd.config.display.eye_target_width_in_pixels; + uint32_t h_pixels = survive->hmd.config.display.eye_target_height_in_pixels; - double fov = 2 * atan2(w_meters - lens_horizontal_separation / 2.0, eye_to_screen_distance); - - for (int view = 0; view < 2; view++) { - survive->distortion[view].aspect_x_over_y = 0.89999997615814209f; - survive->distortion[view].grow_for_undistort = 0.5f; - survive->distortion[view].undistort_r2_cutoff = 1.0f; - } - - survive->hmd.rot[0].w = 1.0f; - survive->hmd.rot[1].w = 1.0f; - - //! @todo: use IPD for FOV - survive->hmd.ipd = 0.063; - survive->hmd.proximity = 0; - - uint16_t w_pixels = 1080; - uint16_t h_pixels = 1200; - const cJSON *device_json = cJSON_GetObjectItemCaseSensitive(json, "device"); - if (device_json) { - if (survive->variant != VIVE_VARIANT_VALVE_INDEX) { - survive->distortion[0].aspect_x_over_y = - _json_get_float(device_json, "physical_aspect_x_over_y"); - survive->distortion[1].aspect_x_over_y = survive->distortion[0].aspect_x_over_y; - - //! @todo: fov calculation needs to be fixed, only works - //! with hardcoded value - // lens_horizontal_separation = _json_get_double(json, - // "lens_separation"); - } - h_pixels = (uint16_t)_json_get_int(device_json, "eye_target_height_in_pixels"); - w_pixels = (uint16_t)_json_get_int(device_json, "eye_target_width_in_pixels"); - } - - const cJSON *eye_transform_json = cJSON_GetObjectItemCaseSensitive(json, "tracking_to_eye_transform"); - if (eye_transform_json) { - for (uint8_t eye = 0; eye < 2; eye++) { - get_distortion_properties(survive, eye_transform_json, eye); - } - } - - SURVIVE_INFO(survive, "Survive eye resolution %dx%d", w_pixels, h_pixels); - - cJSON_Delete(json); + SURVIVE_DEBUG(survive, "display: %dx%d", w_pixels, h_pixels); // Main display. survive->base.hmd->screens[0].w_pixels = (int)w_pixels * 2; survive->base.hmd->screens[0].h_pixels = (int)h_pixels; - if (survive->variant == VIVE_VARIANT_VALVE_INDEX) + if (survive->hmd.config.variant == VIVE_VARIANT_INDEX) { + lens_horizontal_separation = 0.06; + h_meters = 0.07; + // eye relief knob adjusts this around [0.0255(near)-0.275(far)] + eye_to_screen_distance = 0.0255; + survive->base.hmd->screens[0].nominal_frame_interval_ns = (uint64_t)time_s_to_ns(1.0f / 144.0f); - else + } else { survive->base.hmd->screens[0].nominal_frame_interval_ns = (uint64_t)time_s_to_ns(1.0f / 90.0f); + } + + double fov = 2 * atan2(w_meters - lens_horizontal_separation / 2.0, eye_to_screen_distance); struct xrt_vec2 lens_center[2]; @@ -1098,6 +918,12 @@ _create_hmd_device(struct survive_system *sys, enum VIVE_VARIANT variant, const survive->base.inputs[0].name = XRT_INPUT_GENERIC_HEAD_POSE; + survive->last_inputs = U_TYPED_ARRAY_CALLOC(struct xrt_input, survive->base.num_inputs); + survive->num_last_inputs = survive->base.num_inputs; + for (size_t i = 0; i < survive->base.num_inputs; i++) { + survive->last_inputs[i] = survive->base.inputs[i]; + } + return true; } @@ -1160,59 +986,74 @@ static struct xrt_binding_profile binding_profiles_vive[1] = { } while (0) static bool -_create_controller_device(struct survive_system *sys, const SurviveSimpleObject *sso, enum VIVE_VARIANT variant) +_create_controller_device(struct survive_system *sys, + const SurviveSimpleObject *sso, + struct vive_controller_config *config) { + enum VIVE_CONTROLLER_VARIANT variant = config->variant; + + int idx = -1; + if (variant == CONTROLLER_VIVE_WAND) { + if (sys->controllers[SURVIVE_LEFT_CONTROLLER_INDEX] == NULL) { + idx = SURVIVE_LEFT_CONTROLLER_INDEX; + } else if (sys->controllers[SURVIVE_RIGHT_CONTROLLER_INDEX] == NULL) { + idx = SURVIVE_RIGHT_CONTROLLER_INDEX; + } else { + U_LOG_IFL_E(sys->ll, "Only creating 2 controllers!"); + return false; + } + } else if (variant == CONTROLLER_INDEX_LEFT) { + if (sys->controllers[SURVIVE_LEFT_CONTROLLER_INDEX] == NULL) { + idx = SURVIVE_LEFT_CONTROLLER_INDEX; + } else { + U_LOG_IFL_E(sys->ll, "Only creating 1 left controller!"); + return false; + } + } else if (variant == CONTROLLER_INDEX_RIGHT) { + if (sys->controllers[SURVIVE_RIGHT_CONTROLLER_INDEX] == NULL) { + idx = SURVIVE_RIGHT_CONTROLLER_INDEX; + } else { + U_LOG_IFL_E(sys->ll, "Only creating 1 right controller!"); + return false; + } + } else if (variant == CONTROLLER_TRACKER_GEN1 || variant == CONTROLLER_TRACKER_GEN2) { + for (int i = SURVIVE_NON_CONTROLLER_START; i < MAX_TRACKED_DEVICE_COUNT; i++) { + if (sys->controllers[i] == NULL) { + idx = i; + break; + } + } + } + + if (idx == -1) { + U_LOG_IFL_E(sys->ll, "Skipping survive device we couldn't assign: %s!", config->firmware.model_number); + return false; + } + enum u_device_alloc_flags flags = 0; int inputs = VIVE_CONTROLLER_MAX_INDEX; int outputs = 1; struct survive_device *survive = U_DEVICE_ALLOCATE(struct survive_device, flags, inputs, outputs); + survive->ctrl.config = *config; + m_relation_history_create(&survive->relation_hist); - int idx = -1; - if (variant == VIVE_VARIANT_HTC_VIVE_CONTROLLER) { - if (sys->controllers[SURVIVE_LEFT_CONTROLLER] == NULL) { - idx = SURVIVE_LEFT_CONTROLLER; - } else if (sys->controllers[SURVIVE_RIGHT_CONTROLLER] == NULL) { - idx = SURVIVE_RIGHT_CONTROLLER; - } else { - SURVIVE_ERROR(survive, "Only creating 2 controllers!"); - return false; - } - } else if (variant == VIVE_VARIANT_VALVE_INDEX_LEFT_CONTROLLER) { - if (sys->controllers[SURVIVE_LEFT_CONTROLLER] == NULL) { - idx = SURVIVE_LEFT_CONTROLLER; - } else { - SURVIVE_ERROR(survive, "Only creating 1 left controller!"); - return false; - } - } else if (variant == VIVE_VARIANT_VALVE_INDEX_RIGHT_CONTROLLER) { - if (sys->controllers[SURVIVE_RIGHT_CONTROLLER] == NULL) { - idx = SURVIVE_RIGHT_CONTROLLER; - } else { - SURVIVE_ERROR(survive, "Only creating 1 right controller!"); - return false; - } - } sys->controllers[idx] = survive; survive->sys = sys; - survive->variant = variant; survive->survive_obj = sso; + survive->device_type = DEVICE_TYPE_CONTROLLER; - survive->num = idx; survive->base.tracking_origin = &sys->base; survive->base.destroy = survive_device_destroy; survive->base.update_inputs = survive_device_update_inputs; survive->base.get_tracked_pose = survive_device_get_tracked_pose; survive->base.set_output = survive_controller_device_set_output; + snprintf(survive->base.serial, XRT_DEVICE_NAME_LEN, "%s", survive->ctrl.config.firmware.device_serial_number); - //! @todo: May use Vive Wands + Index HMDs or Index Controllers + Vive - //! HMD - if (variant == VIVE_VARIANT_VALVE_INDEX_LEFT_CONTROLLER || - variant == VIVE_VARIANT_VALVE_INDEX_RIGHT_CONTROLLER) { + if (variant == CONTROLLER_INDEX_LEFT || variant == CONTROLLER_INDEX_RIGHT) { survive->base.name = XRT_DEVICE_INDEX_CONTROLLER; - snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "Survive Valve Index Controller %d", idx); SET_INDEX_INPUT(SYSTEM_CLICK, SYSTEM_CLICK); SET_INDEX_INPUT(A_CLICK, A_CLICK); @@ -1236,20 +1077,20 @@ _create_controller_device(struct survive_system *sys, const SurviveSimpleObject SET_INDEX_INPUT(AIM_POSE, AIM_POSE); SET_INDEX_INPUT(GRIP_POSE, GRIP_POSE); - if (variant == VIVE_VARIANT_VALVE_INDEX_LEFT_CONTROLLER) { + if (variant == CONTROLLER_INDEX_LEFT) { survive->base.device_type = XRT_DEVICE_TYPE_LEFT_HAND_CONTROLLER; survive->base.inputs[VIVE_CONTROLLER_HAND_TRACKING].name = XRT_INPUT_GENERIC_HAND_TRACKING_LEFT; - } else if (variant == VIVE_VARIANT_VALVE_INDEX_RIGHT_CONTROLLER) { + snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "Valve Index Left Controller (libsurvive)"); + } else if (variant == CONTROLLER_INDEX_RIGHT) { survive->base.device_type = XRT_DEVICE_TYPE_RIGHT_HAND_CONTROLLER; survive->base.inputs[VIVE_CONTROLLER_HAND_TRACKING].name = XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT; - } else { - survive->base.device_type = XRT_DEVICE_TYPE_ANY_HAND_CONTROLLER; + snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "Valve Index Right Controller (libsurvive)"); } survive->base.get_hand_tracking = survive_controller_get_hand_tracking; - enum xrt_hand hand = idx == SURVIVE_LEFT_CONTROLLER ? XRT_HAND_LEFT : XRT_HAND_RIGHT; + enum xrt_hand hand = idx == SURVIVE_LEFT_CONTROLLER_INDEX ? XRT_HAND_LEFT : XRT_HAND_RIGHT; u_hand_joints_init_default_set(&survive->ctrl.hand_tracking, hand, XRT_HAND_TRACKING_MODEL_FINGERL_CURL, 1.0); @@ -1260,9 +1101,9 @@ _create_controller_device(struct survive_system *sys, const SurviveSimpleObject survive->base.hand_tracking_supported = true; - } else if (survive->variant == VIVE_VARIANT_HTC_VIVE_CONTROLLER) { + } else if (survive->ctrl.config.variant == CONTROLLER_VIVE_WAND) { survive->base.name = XRT_DEVICE_VIVE_WAND; - snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "Survive Vive Wand Controller %d", idx); + snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "Vive Wand Controller (libsurvive)"); SET_WAND_INPUT(SYSTEM_CLICK, SYSTEM_CLICK); SET_WAND_INPUT(SQUEEZE_CLICK, SQUEEZE_CLICK); @@ -1282,75 +1123,197 @@ _create_controller_device(struct survive_system *sys, const SurviveSimpleObject survive->base.num_binding_profiles = ARRAY_SIZE(binding_profiles_vive); survive->base.device_type = XRT_DEVICE_TYPE_ANY_HAND_CONTROLLER; + } else if (survive->ctrl.config.variant == CONTROLLER_TRACKER_GEN1 || + survive->ctrl.config.variant == CONTROLLER_TRACKER_GEN2) { + if (survive->ctrl.config.variant == CONTROLLER_TRACKER_GEN1) { + survive->base.name = XRT_DEVICE_VIVE_TRACKER_GEN1; + snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "Vive Tracker Gen1 (libsurvive)"); + } else if (survive->ctrl.config.variant == CONTROLLER_TRACKER_GEN2) { + survive->base.name = XRT_DEVICE_VIVE_TRACKER_GEN2; + snprintf(survive->base.str, XRT_DEVICE_NAME_LEN, "Vive Tracker Gen2 (libsurvive)"); + } + + survive->base.device_type = XRT_DEVICE_TYPE_GENERIC_TRACKER; + + survive->base.inputs[VIVE_TRACKER_POSE].name = XRT_INPUT_GENERIC_TRACKER_POSE; } survive->base.orientation_tracking_supported = true; survive->base.position_tracking_supported = true; + survive->last_inputs = U_TYPED_ARRAY_CALLOC(struct xrt_input, survive->base.num_inputs); + survive->num_last_inputs = survive->base.num_inputs; + for (size_t i = 0; i < survive->base.num_inputs; i++) { + survive->last_inputs[i] = survive->base.inputs[i]; + } + SURVIVE_DEBUG(survive, "Created Controller %d", idx); return true; } DEBUG_GET_ONCE_LOG_OPTION(survive_log, "SURVIVE_LOG", U_LOGGING_WARN) +DEBUG_GET_ONCE_OPTION(survive_lh_gen, "SURVIVE_LH_GEN", "0") -static enum VIVE_VARIANT -_product_to_variant(uint16_t product_id) +static void +add_device(struct survive_system *ss, const struct SurviveSimpleConfigEvent *e) { - switch (product_id) { - case VIVE_PID: return VIVE_VARIANT_VIVE; - case VIVE_PRO_MAINBOARD_PID: return VIVE_VARIANT_PRO; - case VIVE_PRO_LHR_PID: return VIVE_VARIANT_VALVE_INDEX; - default: U_LOG_W("No product ids matched %.4x", product_id); return VIVE_UNKNOWN; + struct SurviveSimpleObject *sso = e->object; + + U_LOG_IFL_D(ss->ll, "Got device config from survive"); + + enum SurviveSimpleObject_type type = survive_simple_object_get_type(sso); + + char *conf_str = (char *)survive_simple_json_config(sso); + + if (type == SurviveSimpleObject_HMD) { + + _create_hmd_device(ss, sso, conf_str); + + } else if (type == SurviveSimpleObject_OBJECT) { + struct vive_controller_config config = {0}; + vive_config_parse_controller(&config, conf_str, ss->ll); + + switch (config.variant) { + case CONTROLLER_VIVE_WAND: + case CONTROLLER_INDEX_LEFT: + case CONTROLLER_INDEX_RIGHT: + case CONTROLLER_TRACKER_GEN1: + case CONTROLLER_TRACKER_GEN2: + U_LOG_IFL_D(ss->ll, "Adding controller: %s.", config.firmware.model_number); + _create_controller_device(ss, sso, &config); + break; + default: + U_LOG_IFL_D(ss->ll, "Skip non controller obj %s.", config.firmware.model_number); + U_LOG_IFL_T(ss->ll, "json: %s", conf_str); + break; + } + } else { + U_LOG_IFL_D(ss->ll, "Skip non OBJECT obj."); } } -#define JSON_STRING(a, b, c) u_json_get_string_into_array(u_json_get(a, b), c, sizeof(c)) - -static enum VIVE_VARIANT -get_variant_from_json(struct survive_system *ss, cJSON *json) +static bool +add_connected_devices(struct survive_system *ss) { - char model_number[32]; + /** @todo We don't know how many device added events we will get. + * After 25ms Index HMD + Controllers are added here. So 250ms should be a safe value. + * Device added just means libsurvive knows the usb devices, the config will then be loaded asynchronously. + */ + os_nanosleep(250 * 1000 * 1000); - if (u_json_get(json, "model_number")) { - JSON_STRING(json, "model_number", model_number); - } else { - JSON_STRING(json, "model_name", model_number); + size_t objs = survive_simple_get_object_count(ss->ctx); + U_LOG_IFL_D(ss->ll, "Object count: %zu", objs); + + timepoint_ns start = os_monotonic_get_ns(); + + // First count how many non-lighthouse objects libsurvive knows. + // Then poll events until we have gotten configs for this many, or until timeout. + int configs_to_wait_for = 0; + int configs_gotten = 0; + + for (const SurviveSimpleObject *sso = survive_simple_get_first_object(ss->ctx); sso; + sso = survive_simple_get_next_object(ss->ctx, sso)) { + enum SurviveSimpleObject_type t = survive_simple_object_get_type(sso); + const char *name = survive_simple_object_name(sso); + U_LOG_IFL_D(ss->ll, "Object name %s: type %d", name, t); + + // we only want to wait for configs of HMDs and controllers / trackers. + // Note: HMDs will be of type SurviveSimpleObject_OBJECT until the config is loaded. + if (t == SurviveSimpleObject_HMD || t == SurviveSimpleObject_OBJECT) { + configs_to_wait_for++; + } } - enum VIVE_VARIANT variant = VIVE_UNKNOWN; + U_LOG_IFL_D(ss->ll, "Waiting for %d configs", configs_to_wait_for); + while (configs_gotten < configs_to_wait_for) { + struct SurviveSimpleEvent event = {0}; + while (survive_simple_next_event(ss->ctx, &event) != SurviveSimpleEventType_None) { + if (event.event_type == SurviveSimpleEventType_ConfigEvent) { + _process_event(ss, &event); + configs_gotten++; + U_LOG_IFL_D(ss->ll, "Got config from device: %d/%d", configs_gotten, + configs_to_wait_for); + } else { + U_LOG_IFL_T(ss->ll, "Skipping event type %d", event.event_type); + } + } - if (strcmp(model_number, "Vive. Controller MV") == 0) { - variant = VIVE_VARIANT_HTC_VIVE_CONTROLLER; - U_LOG_D("Found Vive Wand controller"); - } else if (strcmp(model_number, "Knuckles Right") == 0) { - variant = VIVE_VARIANT_VALVE_INDEX_RIGHT_CONTROLLER; - U_LOG_D("Found Knuckles Right controller"); - } else if (strcmp(model_number, "Knuckles Left") == 0) { - variant = VIVE_VARIANT_VALVE_INDEX_LEFT_CONTROLLER; - U_LOG_D("Found Knuckles Left controller"); - } else if (strcmp(model_number, "Vive Tracker PVT") == 0) { - variant = VIVE_VARIANT_TRACKER_V1; - U_LOG_D("Found Gen 1 tracker."); - } else if (strcmp(model_number, "VIVE Tracker Pro MV") == 0) { - variant = VIVE_VARIANT_TRACKER_v2; - U_LOG_D("Found Gen 2 tracker."); - } else if (strcmp(model_number, "Utah MP") == 0) { - U_LOG_W("Found Utah MP (Index HMD), not a controller!"); - } else { - U_LOG_E("Failed to parse controller variant: %s", model_number); + if (time_ns_to_s(os_monotonic_get_ns() - start) > ss->wait_timeout) { + U_LOG_IFL_D(ss->ll, "Timed out after getting configs for %d/%d devices", configs_gotten, + configs_to_wait_for); + break; + } + os_nanosleep(500 * 1000); + } + U_LOG_IFL_D(ss->ll, "Waiting for configs took %f ms", time_ns_to_ms_f(os_monotonic_get_ns() - start)); + return true; +} + +static void * +run_event_thread(void *ptr) +{ + struct survive_system *ss = (struct survive_system *)ptr; + + os_thread_helper_lock(&ss->event_thread); + while (os_thread_helper_is_running_locked(&ss->event_thread)) { + os_thread_helper_unlock(&ss->event_thread); + + // one event queue for all devices. _process_events() updates all devices + struct SurviveSimpleEvent event = {0}; + survive_simple_wait_for_event(ss->ctx, &event); + + os_mutex_lock(&ss->lock); + _process_event(ss, &event); + os_mutex_unlock(&ss->lock); + + // Just keep swimming. + os_thread_helper_lock(&ss->event_thread); } - return variant; + os_thread_helper_unlock(&ss->event_thread); + + return NULL; +} + + +static void +survive_get_user_config(struct survive_system *ss) +{ + // Set defaults + ss->wait_timeout = DEFAULT_WAIT_TIMEOUT; + + // Open and parse file + struct u_config_json wrap = {0}; + u_config_json_open_or_create_main_file(&wrap); + if (!wrap.file_loaded) { + return; + } + + // Find the dict we care about in the file. It might not be there; if so return early. + cJSON *config_json = cJSON_GetObjectItemCaseSensitive(wrap.root, "config_survive"); + if (config_json == NULL) { + return; + } + + // Get wait timeout key. No need to null-check - read u_json_get_float. + cJSON *wait_timeout = cJSON_GetObjectItemCaseSensitive(config_json, "wait_timeout"); + u_json_get_float(wait_timeout, &ss->wait_timeout); + + // Log + U_LOG_D("Wait timeout is %f seconds!", ss->wait_timeout); + + // Clean up after ourselves + cJSON_Delete(wrap.root); + return; } int -survive_found(struct xrt_prober *xp, - struct xrt_prober_device **devices, - size_t num_devices, - size_t index, - cJSON *attached_data, - struct xrt_device **out_xdevs) +survive_device_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs) { if (survive_already_initialized) { U_LOG_I( @@ -1362,7 +1325,7 @@ survive_found(struct xrt_prober *xp, SurviveSimpleContext *actx = NULL; #if 1 char *survive_args[] = { - "Monado-libsurvive", + "Monado-libsurvive", "--lighthouse-gen", (char *)debug_get_option_survive_lh_gen(), //"--time-window", "1500000" //"--use-imu", "0", //"--use-kalman", "0" @@ -1379,8 +1342,6 @@ survive_found(struct xrt_prober *xp, struct survive_system *ss = U_TYPED_CALLOC(struct survive_system); - struct xrt_prober_device *dev = devices[index]; - survive_simple_start_thread(actx); ss->ctx = actx; @@ -1393,52 +1354,11 @@ survive_found(struct xrt_prober *xp, ss->ll = debug_get_log_option_survive_log(); - /* iterate over all devices, if SurviveSimpleObject_OBJECT parse config - * and if controller, add it with variant from config. - */ - for (const SurviveSimpleObject *it = survive_simple_get_first_object(ss->ctx); it != 0; - it = survive_simple_get_next_object(ss->ctx, it)) { + survive_get_user_config(ss); - if (!wait_for_device_config(it)) { - U_LOG_IFL_E(ss->ll, "Failed to get device config from survive"); - continue; - } - - U_LOG_IFL_D(ss->ll, "Got device config from survive"); - - enum SurviveSimpleObject_type type = survive_simple_object_get_type(it); - - if (type == SurviveSimpleObject_HMD) { - enum VIVE_VARIANT variant = _product_to_variant(dev->product_id); - U_LOG_I("survive HMD: Assuming variant %d", variant); - _create_hmd_device(ss, variant, it); - } else if (type == SurviveSimpleObject_OBJECT) { - char *json_string = survive_get_json_config(it); - cJSON *json = cJSON_Parse(json_string); - if (!cJSON_IsObject(json)) { - U_LOG_IFL_E(ss->ll, "Could not parse JSON data."); - cJSON_Delete(json); - continue; - } - - enum VIVE_VARIANT variant = get_variant_from_json(ss, json); - - switch (variant) { - case VIVE_VARIANT_HTC_VIVE_CONTROLLER: - case VIVE_VARIANT_VALVE_INDEX_LEFT_CONTROLLER: - case VIVE_VARIANT_VALVE_INDEX_RIGHT_CONTROLLER: - U_LOG_IFL_D(ss->ll, "Adding controller."); - _create_controller_device(ss, it, variant); - break; - default: - U_LOG_IFL_D(ss->ll, "Skip non controller obj."); - U_LOG_IFL_T(ss->ll, "json: %s", json_string); - break; - } - cJSON_Delete(json); - } else { - U_LOG_IFL_D(ss->ll, "Skip non OBJECT obj."); - } + while (!add_connected_devices(ss)) { + U_LOG_IFL_E(ss->ll, "Failed to get device config from survive"); + continue; } // U_LOG_D("Survive HMD %p, controller %p %p", (void *)ss->hmd, @@ -1451,16 +1371,69 @@ survive_found(struct xrt_prober *xp, } int out_idx = 0; - if (ss->hmd) { + if (ss->hmd && !no_hmds) { out_xdevs[out_idx++] = &ss->hmd->base; } - if (&ss->controllers[SURVIVE_LEFT_CONTROLLER]) { - out_xdevs[out_idx++] = &ss->controllers[SURVIVE_LEFT_CONTROLLER]->base; - } - if (&ss->controllers[SURVIVE_LEFT_CONTROLLER]) { - out_xdevs[out_idx++] = &ss->controllers[SURVIVE_RIGHT_CONTROLLER]->base; + + bool found_controllers = false; + + for (int i = 0; i < MAX_TRACKED_DEVICE_COUNT; i++) { + + if (out_idx == XRT_MAX_DEVICES_PER_PROBE - 1) { + U_LOG_IFL_W(ss->ll, "Probed max of %d devices, ignoring further devices", + XRT_MAX_DEVICES_PER_PROBE); + return out_idx; + } + + if (ss->controllers[i] != NULL) { + out_xdevs[out_idx++] = &ss->controllers[i]->base; + found_controllers = true; + } } +#ifdef XRT_BUILD_DRIVER_HANDTRACKING + // We want to hit this codepath when we find a HMD but no controllers. + if ((ss->hmd != NULL) && !found_controllers) { + struct t_stereo_camera_calibration *cal = NULL; + + struct xrt_pose head_in_left_cam; + vive_get_stereo_camera_calibration(&ss->hmd->hmd.config, &cal, &head_in_left_cam); + + struct xrt_device *ht = ht_device_create(xp, cal); + if (ht != NULL) { // Returns NULL if there's a problem and the hand tracker can't start. By no means a + // fatal error. + struct xrt_device *wrap = + multi_create_tracking_override(XRT_TRACKING_OVERRIDE_ATTACHED, ht, &ss->hmd->base, + XRT_INPUT_GENERIC_HEAD_POSE, &head_in_left_cam); + out_xdevs[out_idx++] = wrap; + } + // Don't need it anymore. And it's not even created unless we hit this codepath, which is somewhat hard. + t_stereo_camera_calibration_reference(&cal, NULL); + } +#endif + + survive_already_initialized = true; + + // Mutex before thread. + int ret = os_mutex_init(&ss->lock); + if (ret != 0) { + U_LOG_IFL_E(ss->ll, "Failed to init mutex!"); + survive_device_destroy((struct xrt_device *)ss->hmd); + for (int i = 0; i < MAX_TRACKED_DEVICE_COUNT; i++) { + survive_device_destroy((struct xrt_device *)ss->controllers[i]); + } + return 0; + } + + ret = os_thread_helper_start(&ss->event_thread, run_event_thread, ss); + if (ret != 0) { + U_LOG_IFL_E(ss->ll, "Failed to start event thread!"); + survive_device_destroy((struct xrt_device *)ss->hmd); + for (int i = 0; i < MAX_TRACKED_DEVICE_COUNT; i++) { + survive_device_destroy((struct xrt_device *)ss->controllers[i]); + } + return 0; + } return out_idx; } diff --git a/src/xrt/drivers/survive/survive_driver.h b/src/xrt/drivers/survive/survive_driver.h new file mode 100644 index 000000000..e43e1f026 --- /dev/null +++ b/src/xrt/drivers/survive/survive_driver.h @@ -0,0 +1,16 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: Apache-2.0 +/*! + * @file + * @brief Adapter to Libsurvive. + * @author Christoph Haag + * @ingroup drv_survive + */ + + +int +survive_device_autoprobe(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs); diff --git a/src/xrt/drivers/survive/survive_interface.h b/src/xrt/drivers/survive/survive_interface.h index fdb94bdf0..a2750dab8 100644 --- a/src/xrt/drivers/survive/survive_interface.h +++ b/src/xrt/drivers/survive/survive_interface.h @@ -14,22 +14,20 @@ extern "C" { #endif -#define HTC_VID 0x0bb4 -#define VALVE_VID 0x28de +/*! + * @defgroup drv_survive Lighthouse tracking using libsurvive + * @ingroup drv + * + * @brief + */ -#define VIVE_PID 0x2c87 -#define VIVE_LIGHTHOUSE_FPGA_RX 0x2000 - -#define VIVE_PRO_MAINBOARD_PID 0x0309 -#define VIVE_PRO_LHR_PID 0x2300 - -int -survive_found(struct xrt_prober *xp, - struct xrt_prober_device **devices, - size_t num_devices, - size_t index, - cJSON *attached_data, - struct xrt_device **out_xdevs); +/*! + * Create a probe for libsurvive + * + * @ingroup drv_survive + */ +struct xrt_auto_prober * +survive_create_auto_prober(); #ifdef __cplusplus } diff --git a/src/xrt/drivers/survive/survive_prober.c b/src/xrt/drivers/survive/survive_prober.c new file mode 100644 index 000000000..04a566710 --- /dev/null +++ b/src/xrt/drivers/survive/survive_prober.c @@ -0,0 +1,51 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief libsurvive prober code. + * @author Christoph Haag + * @ingroup drv_survive + */ + + +#include "xrt/xrt_prober.h" + +#include "util/u_misc.h" + +#include "survive_interface.h" +#include "survive_driver.h" + +/*! + * @implements xrt_auto_prober + */ +struct survive_prober +{ + struct xrt_auto_prober base; +}; + +//! @private @memberof survive_prober +static inline struct survive_prober * +survive_prober(struct xrt_auto_prober *p) +{ + return (struct survive_prober *)p; +} + +//! @public @memberof survive_prober +static void +survive_prober_destroy(struct xrt_auto_prober *p) +{ + struct survive_prober *survive_p = survive_prober(p); + + free(survive_p); +} + +struct xrt_auto_prober * +survive_create_auto_prober() +{ + struct survive_prober *survive_p = U_TYPED_CALLOC(struct survive_prober); + survive_p->base.name = "survive"; + survive_p->base.destroy = survive_prober_destroy; + survive_p->base.lelo_dallas_autoprobe = survive_device_autoprobe; + + return &survive_p->base; +} diff --git a/src/xrt/drivers/survive/survive_wrap.c b/src/xrt/drivers/survive/survive_wrap.c deleted file mode 100644 index a58f4586b..000000000 --- a/src/xrt/drivers/survive/survive_wrap.c +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2020, Collabora, Ltd. -// SPDX-License-Identifier: BSL-1.0 -/*! - * @file - * @brief low level libsurvive wrapper - * @author Christoph Haag - * @ingroup drv_survive - */ - -#define SURVIVE_ENABLE_FULL_API 1 -#include "survive_api.h" - -#include "survive_wrap.h" - - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wignored-qualifiers" - -// TODO: expose config values we need through actual survive API -#include "survive.h" - -#pragma GCC diagnostic pop - -bool -survive_has_obj(const SurviveSimpleObject *sso) -{ - SurviveObject *so = survive_simple_get_survive_object(sso); - return so != NULL; -} - -bool -survive_config_ready(const SurviveSimpleObject *sso) -{ - SurviveObject *so = survive_simple_get_survive_object(sso); - return so && so->conf != 0; -} - -char * -survive_get_json_config(const SurviveSimpleObject *sso) -{ - SurviveObject *so = survive_simple_get_survive_object(sso); - return so->conf; -} diff --git a/src/xrt/drivers/survive/survive_wrap.h b/src/xrt/drivers/survive/survive_wrap.h deleted file mode 100644 index 9bf69e28b..000000000 --- a/src/xrt/drivers/survive/survive_wrap.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020, Collabora, Ltd. -// SPDX-License-Identifier: BSL-1.0 -/*! - * @file - * @brief low level libsurvive wrapper - * @author Christoph Haag - * @ingroup drv_survive - */ - -#pragma once - -#include "survive_api.h" - -bool -survive_has_obj(const SurviveSimpleObject *sso); - -bool -survive_config_ready(const SurviveSimpleObject *sso); - -char * -survive_get_json_config(const SurviveSimpleObject *sso); diff --git a/src/xrt/drivers/ultraleap_v2/readme.md b/src/xrt/drivers/ultraleap_v2/readme.md new file mode 100644 index 000000000..7a7a87a2c --- /dev/null +++ b/src/xrt/drivers/ultraleap_v2/readme.md @@ -0,0 +1,36 @@ +# About Monado's UltraLeap driver + + + +## Building + +To build you need `Leap.h` and `LeapMath.h` in `/usr/local/include`; and +`libLeap.so` in `/usr/local/lib`, and this should automatically build. + +## Running + +To have the ultraleap driver successfully initialize, you need to have the Leap +Motion Controller plugged in, and `leapd` running. Running `sudo leapd` in +another terminal works but it may be slightly more convenient to have it run as +a systemd service. + +## Configuring + +Presumably, you're using this driver because you want to stick the Leap Motion +Controller on the front of your HMD and have it track your hands. + +If you don't have a config file at `~/.config/monado/config_v0.json` (or +wherever you set `XDG_CONFIG_DIR`), your tracked hands will show up near the +tracking origin and not move around with your HMD, which is probably not what +you want. + +Instead you probably want to configure Monado to make your Leap Motion +Controller-tracked hands follow around your HMD. There's an example of how to do +this with North Star in `doc/example_configs/config_v0.northstar_lonestar.json`. +If you're using a North Star headset, that should work but unless you're using +the Lone Star NS variant you'll need to edit the offsets. If you're using some +other HMD you'll have to edit the `tracker_device_serial` to be your HMD serial, +and your own offsets. diff --git a/src/xrt/drivers/ultraleap_v2/ulv2_driver.cpp b/src/xrt/drivers/ultraleap_v2/ulv2_driver.cpp new file mode 100644 index 000000000..0dfae0b6c --- /dev/null +++ b/src/xrt/drivers/ultraleap_v2/ulv2_driver.cpp @@ -0,0 +1,452 @@ +// Copyright 2020-2021, Collabora, Ltd. +// Copyright 2020-2021, Moses Turner +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Driver for Ultraleap's V2 API for the Leap Motion Controller. + * @author Moses Turner + * @author Christoph Haag + * @ingroup drv_ulv2 + */ + +#include "ulv2_interface.h" +#include "util/u_device.h" +#include "util/u_var.h" +#include "util/u_debug.h" +#include "math/m_space.h" +#include "math/m_api.h" +#include "util/u_time.h" +#include "os/os_time.h" +#include "os/os_threading.h" + +#include "Leap.h" + +DEBUG_GET_ONCE_LOG_OPTION(ulv2_log, "ULV2_LOG", U_LOGGING_INFO) + +#define ULV2_TRACE(ulv2d, ...) U_LOG_XDEV_IFL_T(&ulv2d->base, ulv2d->ll, __VA_ARGS__) +#define ULV2_DEBUG(ulv2d, ...) U_LOG_XDEV_IFL_D(&ulv2d->base, ulv2d->ll, __VA_ARGS__) +#define ULV2_INFO(ulv2d, ...) U_LOG_XDEV_IFL_I(&ulv2d->base, ulv2d->ll, __VA_ARGS__) +#define ULV2_WARN(ulv2d, ...) U_LOG_XDEV_IFL_W(&ulv2d->base, ulv2d->ll, __VA_ARGS__) +#define ULV2_ERROR(ulv2d, ...) U_LOG_XDEV_IFL_E(&ulv2d->base, ulv2d->ll, __VA_ARGS__) + +#define printf_pose(pose) \ + printf("%f %f %f %f %f %f %f\n", pose.position.x, pose.position.y, pose.position.z, pose.orientation.x, \ + pose.orientation.y, pose.orientation.z, pose.orientation.w); +extern "C" { + +enum xrt_space_relation_flags valid_flags = (enum xrt_space_relation_flags)( + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | + XRT_SPACE_RELATION_POSITION_VALID_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT); + +// Excuse my sketchy thread stuff, I'm sure this violates all kinds of best practices. It uusssuallyyy doesn't explode. +// -Moses Turner +enum leap_thread_status +{ + THREAD_NOT_STARTED, + THREAD_OK, + THREAD_ERRORED_OUT, +}; + +struct ulv2_device +{ + struct xrt_device base; + + struct xrt_tracking_origin tracking_origin; + + enum u_logging_level ll; + + bool pthread_should_stop; + + enum leap_thread_status our_thread_status; + + struct os_thread_helper leap_loop_oth; + + struct xrt_hand_joint_set joints_write_in[2]; + + // Only read/write these last two if you have the mutex + struct xrt_hand_joint_set joints_read_out[2]; + + bool hand_exists[2]; +}; + +inline struct ulv2_device * +ulv2_device(struct xrt_device *xdev) +{ + return (struct ulv2_device *)xdev; +} + + +// Roughly, take the Leap hand joint, do some coordinate conversions, and save it in a xrt_hand_joint_value +static void +ulv2_process_joint( + Leap::Vector jointpos, Leap::Matrix jointbasis, float width, int side, struct xrt_hand_joint_value *joint) +{ + struct xrt_space_relation *relation = &joint->relation; + joint->radius = (width / 1000) / 2; + + struct xrt_matrix_3x3 turn_into_quat; + // clang-format off + // These are matrices so we want to preserve where the rows and columns are, hence the clang-format off + if (side == 1) + { + turn_into_quat = {-jointbasis.xBasis.x, -jointbasis.yBasis.x, -jointbasis.zBasis.x, + -jointbasis.xBasis.z, -jointbasis.yBasis.z, -jointbasis.zBasis.z, + -jointbasis.xBasis.y, -jointbasis.yBasis.y, -jointbasis.zBasis.y}; + } + else + { + turn_into_quat = {jointbasis.xBasis.x, -jointbasis.yBasis.x, -jointbasis.zBasis.x, + jointbasis.xBasis.z, -jointbasis.yBasis.z, -jointbasis.zBasis.z, + jointbasis.xBasis.y, -jointbasis.yBasis.y, -jointbasis.zBasis.y}; + } + // clang-format on + + + math_quat_from_matrix_3x3(&turn_into_quat, &relation->pose.orientation); + relation->pose.position.x = jointpos.x * -1 / 1000.0; + relation->pose.position.y = jointpos.z * -1 / 1000.0; + relation->pose.position.z = jointpos.y * -1 / 1000.0; + relation->relation_flags = valid_flags; +} + +static int error_time; // Lazy counter to prevent printing error messages at 120hz + + +void +ulv2_process_hand(Leap::Hand hand, xrt_hand_joint_set *joint_set, int hi) +{ + +#define xrtj(y) &joint_set->values.hand_joint_set_default[XRT_HAND_JOINT_##y] + + ulv2_process_joint(hand.palmPosition(), hand.basis(), 50, hi, xrtj(PALM)); + ulv2_process_joint(hand.wristPosition(), hand.arm().basis(), 50, hi, xrtj(WRIST)); + + const Leap::FingerList fingers = hand.fingers(); + + // Bunch of macros to make the following + // boilerplate easier to deal with + +#define fb(y) finger.bone(y) +#define prevJ(y) finger.bone(y).prevJoint() +#define nextJ(y) finger.bone(y).nextJoint() + +#define lm Leap::Bone::TYPE_METACARPAL +#define lp Leap::Bone::TYPE_PROXIMAL +#define li Leap::Bone::TYPE_INTERMEDIATE +#define ld Leap::Bone::TYPE_DISTAL + + for (Leap::FingerList::const_iterator fl = fingers.begin(); fl != fingers.end(); ++fl) { + // Iterate on the list of fingers Leap gives us + const Leap::Finger finger = *fl; + Leap::Finger::Type fingerType = finger.type(); + switch (fingerType) { + case Leap::Finger::Type::TYPE_THUMB: + // If the finger is a thumb, then for each joint: process the Leap joint location, + // rotation matrix, finger width, hand side (0 if left, 1 if right), + // and write the finger radius and powe out to the correct xrt_hand_joint_value. + ulv2_process_joint(prevJ(lp), fb(lp).basis(), fb(lp).width(), hi, xrtj(THUMB_METACARPAL)); + ulv2_process_joint(prevJ(li), fb(li).basis(), fb(lp).width(), hi, xrtj(THUMB_PROXIMAL)); + ulv2_process_joint(prevJ(ld), fb(ld).basis(), fb(li).width(), hi, xrtj(THUMB_DISTAL)); + ulv2_process_joint(nextJ(ld), fb(ld).basis(), fb(ld).width(), hi, xrtj(THUMB_TIP)); + // Note that there are only four joints here as opposed to all the other fingers which have five + // joints. + break; + case Leap::Finger::Type::TYPE_INDEX: + // If the finger is an index finger, do the same thing but with index joints + ulv2_process_joint(prevJ(lm), fb(lm).basis(), fb(lm).width(), hi, xrtj(INDEX_METACARPAL)); + ulv2_process_joint(prevJ(lp), fb(lp).basis(), fb(lm).width(), hi, xrtj(INDEX_PROXIMAL)); + ulv2_process_joint(prevJ(li), fb(li).basis(), fb(lp).width(), hi, xrtj(INDEX_INTERMEDIATE)); + ulv2_process_joint(prevJ(ld), fb(ld).basis(), fb(li).width(), hi, xrtj(INDEX_DISTAL)); + ulv2_process_joint(nextJ(ld), fb(ld).basis(), fb(ld).width(), hi, xrtj(INDEX_TIP)); + break; + case Leap::Finger::Type::TYPE_MIDDLE: + // If the finger is a middle finger, do the same thing but with middle joints + ulv2_process_joint(prevJ(lm), fb(lm).basis(), fb(lm).width(), hi, xrtj(MIDDLE_METACARPAL)); + ulv2_process_joint(prevJ(lp), fb(lp).basis(), fb(lm).width(), hi, xrtj(MIDDLE_PROXIMAL)); + ulv2_process_joint(prevJ(li), fb(li).basis(), fb(lp).width(), hi, xrtj(MIDDLE_INTERMEDIATE)); + ulv2_process_joint(prevJ(ld), fb(ld).basis(), fb(li).width(), hi, xrtj(MIDDLE_DISTAL)); + ulv2_process_joint(nextJ(ld), fb(ld).basis(), fb(ld).width(), hi, xrtj(MIDDLE_TIP)); + break; + case Leap::Finger::Type::TYPE_RING: + // Ad nauseum. + ulv2_process_joint(prevJ(lm), fb(lm).basis(), fb(lm).width(), hi, xrtj(RING_METACARPAL)); + ulv2_process_joint(prevJ(lp), fb(lp).basis(), fb(lm).width(), hi, xrtj(RING_PROXIMAL)); + ulv2_process_joint(prevJ(li), fb(li).basis(), fb(lp).width(), hi, xrtj(RING_INTERMEDIATE)); + ulv2_process_joint(prevJ(ld), fb(ld).basis(), fb(li).width(), hi, xrtj(RING_DISTAL)); + ulv2_process_joint(nextJ(ld), fb(ld).basis(), fb(ld).width(), hi, xrtj(RING_TIP)); + break; + case Leap::Finger::Type::TYPE_PINKY: + ulv2_process_joint(prevJ(lm), fb(lm).basis(), fb(lm).width(), hi, xrtj(LITTLE_METACARPAL)); + ulv2_process_joint(prevJ(lp), fb(lp).basis(), fb(lm).width(), hi, xrtj(LITTLE_PROXIMAL)); + ulv2_process_joint(prevJ(li), fb(li).basis(), fb(lp).width(), hi, xrtj(LITTLE_INTERMEDIATE)); + ulv2_process_joint(prevJ(ld), fb(ld).basis(), fb(li).width(), hi, xrtj(LITTLE_DISTAL)); + ulv2_process_joint(nextJ(ld), fb(ld).basis(), fb(ld).width(), hi, xrtj(LITTLE_TIP)); + break; + // I hear that Sagittarius has a better api, in C even, so hopefully + // there'll be less weird boilerplate whenever we get access to that + } + } +} + +void * +leap_input_loop(void *ptr_to_xdev) +{ + float retry_sleep_time = 0.05; + float timeout = 0.5; + int num_tries = (timeout / retry_sleep_time); + bool succeeded_connected = false; + bool succeeded_service_connected = false; + + struct xrt_device *xdev = (struct xrt_device *)ptr_to_xdev; + struct ulv2_device *ulv2d = ulv2_device(xdev); + ULV2_DEBUG(ulv2d, "num tries %d; sleep time %f", num_tries, timeout); + + Leap::Controller LeapController; + os_nanosleep(U_1_000_000_000 * 0.01); + // sleep for an arbitrary amount of time so that Leap::Controller can initialize and connect to the service. + float start = (float)os_monotonic_get_ns() / (float)U_1_000_000_000; + // would be nice to do this only if we're not building release ^^. don't know how to do that though. + + for (int i = 0; i < num_tries; i++) { + succeeded_connected = LeapController.isConnected(); + succeeded_service_connected = LeapController.isServiceConnected(); + if (succeeded_connected) { + ULV2_INFO(ulv2d, "Leap Motion Controller connected!"); + break; + } + if (succeeded_service_connected) { + ULV2_INFO(ulv2d, + "Connected to Leap service, but not " + "connected to Leap Motion " + "controller. Retrying (%i / %i)", + i, num_tries); + // This codepath should very rarely get hit as nowadays this gets probed by VID/PID, so you'd + // have to be pretty fast to unplug after it gets probed and before this check. + } else { + ULV2_INFO(ulv2d, + "Not connected to Leap service. " + "Retrying (%i / %i)", + i, num_tries); + } + os_nanosleep(U_1_000_000_000 * retry_sleep_time); // 1 second * retry_sleep_time + } + + ULV2_DEBUG(ulv2d, "Waited %f seconds", ((float)os_monotonic_get_ns() / (float)U_1_000_000_000) - start); + + bool hmdpolicyset = false; + + + if (!succeeded_connected) { + if (succeeded_service_connected) { + ULV2_INFO(ulv2d, + "Connected to Leap service, but couldn't " + "connect to leap motion controller.\n" + "Is it plugged in and has your Leap service " + "detected it?"); + // ditto on this codepath + } else { + ULV2_INFO(ulv2d, + "Couldn't connect to Leap service. Try " + "running sudo leapd in another terminal."); + } + goto cleanup_leap_loop; + } + + // Try to let the Leap service know that we are on a HMD, not on a desk. + for (int i = 0; i < num_tries; i++) { + LeapController.setPolicy(Leap::Controller::POLICY_OPTIMIZE_HMD); + os_nanosleep(time_s_to_ns(0.02)); + LeapController.setPolicy(Leap::Controller::POLICY_OPTIMIZE_HMD); + hmdpolicyset = LeapController.isPolicySet(Leap::Controller::POLICY_OPTIMIZE_HMD); + if (!hmdpolicyset) { + ULV2_ERROR(ulv2d, "Couldn't set HMD policy. Retrying (%i / %i)", i, num_tries); + } else { + ULV2_DEBUG(ulv2d, "HMD policy set."); + break; + } + os_nanosleep(U_1_000_000_000 * retry_sleep_time); // 1 second * retry_sleep_time + } + ULV2_TRACE(ulv2d, "thread OK\n"); + ulv2d->our_thread_status = THREAD_OK; + + + // Main loop + while (!ulv2d->pthread_should_stop) { + + if (!LeapController.isConnected()) { + if ((error_time % 100) == 0) + ULV2_ERROR(ulv2d, "LeapController is not connected\n"); + os_nanosleep(U_1_000_000_000 * 0.1); + error_time += 1; + continue; + } + error_time = 100; // if there's an error next time, the modulo will return 0. + + Leap::Frame frame = LeapController.frame(); + Leap::HandList hands = frame.hands(); + bool leftbeendone = false; + bool rightbeendone = false; + for (Leap::HandList::const_iterator hl = hands.begin(); hl != hands.end(); ++hl) { + int hi; // hand index + const Leap::Hand hand = *hl; + // if (hand.confidence() < *hand_min_confidence) + // continue; + if (hand.isLeft()) { + if (leftbeendone) + continue; // in case there are more than one left hand + leftbeendone = true; + hi = 0; + } else if (hand.isRight()) { + if (rightbeendone) + continue; // in case there are more than one right hand + rightbeendone = true; + hi = 1; + } else { + continue; + } + + ulv2_process_hand(hand, &ulv2d->joints_write_in[hi], hi); + } + os_thread_helper_lock(&ulv2d->leap_loop_oth); + memcpy(&ulv2d->joints_read_out, &ulv2d->joints_write_in, sizeof(struct xrt_hand_joint_set) * 2); + //! @todo (Moses Turner) Could be using LeapController.now() to try to emulate our own pose prediction, + //! but I ain't got time for that + ulv2d->hand_exists[0] = leftbeendone; + ulv2d->hand_exists[1] = rightbeendone; + os_thread_helper_unlock(&ulv2d->leap_loop_oth); + } + +cleanup_leap_loop: + ULV2_TRACE(ulv2d, "leaving input thread\n"); + ulv2d->our_thread_status = THREAD_ERRORED_OUT; + pthread_exit(NULL); +} + +static void +ulv2_device_update_inputs(struct xrt_device *xdev) +{ + // Empty +} + + +static void +ulv2_device_get_hand_tracking(struct xrt_device *xdev, + enum xrt_input_name name, + uint64_t at_timestamp_ns, + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp_ns) +{ + struct ulv2_device *ulv2d = ulv2_device(xdev); + + if (name != XRT_INPUT_GENERIC_HAND_TRACKING_LEFT && name != XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT) { + ULV2_ERROR(ulv2d, "unknown input name for hand tracker"); + return; + } + + bool hand_index = (name == XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT); // 0 if left, 1 if right. + + bool hand_valid = ulv2d->hand_exists[hand_index]; + + os_thread_helper_lock(&ulv2d->leap_loop_oth); + memcpy(out_value, &ulv2d->joints_read_out[hand_index], sizeof(struct xrt_hand_joint_set)); + hand_valid = ulv2d->hand_exists[hand_index]; + os_thread_helper_unlock(&ulv2d->leap_loop_oth); + m_space_relation_ident(&out_value->hand_pose); + + if (hand_valid) { + out_value->is_active = true; + out_value->hand_pose.relation_flags = valid_flags; + } else { + out_value->is_active = false; + } + // This is a lie - this driver does no pose-prediction or history. Patches welcome. + *out_timestamp_ns = at_timestamp_ns; +} + +static void +ulv2_device_destroy(struct xrt_device *xdev) +{ + struct ulv2_device *ulv2d = ulv2_device(xdev); + + ulv2d->pthread_should_stop = true; + os_thread_helper_stop(&ulv2d->leap_loop_oth); + + // Remove the variable tracking. + u_var_remove_root(ulv2d); + + u_device_free(&ulv2d->base); +} + +int +ulv2_found(struct xrt_prober *xp, + struct xrt_prober_device **devices, + size_t num_devices, + size_t index, + cJSON *attached_data, + struct xrt_device **out_xdev) +{ + enum u_device_alloc_flags flags = U_DEVICE_ALLOC_NO_FLAGS; + + int num_hands = 2; + + struct ulv2_device *ulv2d = U_DEVICE_ALLOCATE(struct ulv2_device, flags, num_hands, 0); + + os_thread_helper_init(&ulv2d->leap_loop_oth); + os_thread_helper_start(&ulv2d->leap_loop_oth, (&leap_input_loop), (void *)&ulv2d->base); + + ulv2d->base.tracking_origin = &ulv2d->tracking_origin; + ulv2d->base.tracking_origin->type = XRT_TRACKING_TYPE_OTHER; + + math_pose_identity(&ulv2d->base.tracking_origin->offset); + + ulv2d->ll = debug_get_log_option_ulv2_log(); + + ulv2d->base.update_inputs = ulv2_device_update_inputs; + ulv2d->base.get_hand_tracking = ulv2_device_get_hand_tracking; + ulv2d->base.destroy = ulv2_device_destroy; + + strncpy(ulv2d->base.str, "Leap Motion v2 driver", XRT_DEVICE_NAME_LEN); + strncpy(ulv2d->base.serial, "Leap Motion v2 driver", XRT_DEVICE_NAME_LEN); + + ulv2d->base.inputs[0].name = XRT_INPUT_GENERIC_HAND_TRACKING_LEFT; + ulv2d->base.inputs[1].name = XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT; + + ulv2d->base.name = XRT_DEVICE_HAND_TRACKER; + + ulv2d->base.device_type = XRT_DEVICE_TYPE_HAND_TRACKER; + ulv2d->base.hand_tracking_supported = true; + + u_var_add_root(ulv2d, "Leap Motion v2 driver", true); + u_var_add_ro_text(ulv2d, ulv2d->base.str, "Name"); + + + + uint64_t start_time = os_monotonic_get_ns(); + uint64_t too_long = time_s_to_ns(15.0f); + + while (ulv2d->our_thread_status != THREAD_OK) { + ULV2_TRACE(ulv2d, "waiting... thread status is %d", ulv2d->our_thread_status); + if (ulv2d->our_thread_status == THREAD_ERRORED_OUT) + goto cleanup; + if ((os_monotonic_get_ns() - (uint64_t)start_time) > too_long) { + ULV2_ERROR(ulv2d, + "For some reason the Leap thread locked up. This is a serious error and should " + "never happen."); + goto cleanup; + } + os_nanosleep(time_s_to_ns(0.01)); + } + + + ULV2_INFO(ulv2d, "Hand Tracker initialized!"); + + out_xdev[0] = &ulv2d->base; + + return 1; + +cleanup: + ulv2_device_destroy(&ulv2d->base); + return 0; +} + +} // extern "C" diff --git a/src/xrt/drivers/ultraleap_v2/ulv2_interface.h b/src/xrt/drivers/ultraleap_v2/ulv2_interface.h new file mode 100644 index 000000000..40c5843db --- /dev/null +++ b/src/xrt/drivers/ultraleap_v2/ulv2_interface.h @@ -0,0 +1,35 @@ +// Copyright 2020-2021, Collabora, Ltd. +// Copyright 2020-2021, Moses Turner. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Driver for Ultraleap's V2 API for the Leap Motion Controller. + * @author Moses Turner + * @author Christoph Haag + * @ingroup drv_ulv2 + */ + +#pragma once + +#include "math/m_api.h" +#include "xrt/xrt_device.h" +#include "xrt/xrt_prober.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ULV2_VID 0xf182 +#define ULV2_PID 0x0003 + +int +ulv2_found(struct xrt_prober *xp, + struct xrt_prober_device **devices, + size_t num_devices, + size_t index, + cJSON *attached_data, + struct xrt_device **out_xdev); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/v4l2/v4l2_driver.c b/src/xrt/drivers/v4l2/v4l2_driver.c index 6c3d818c3..b92abffc1 100644 --- a/src/xrt/drivers/v4l2/v4l2_driver.c +++ b/src/xrt/drivers/v4l2/v4l2_driver.c @@ -1,4 +1,4 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -11,10 +11,12 @@ #include "os/os_time.h" #include "util/u_var.h" +#include "util/u_sink.h" #include "util/u_misc.h" #include "util/u_debug.h" #include "util/u_format.h" #include "util/u_logging.h" +#include "util/u_trace_marker.h" #include "v4l2_interface.h" @@ -68,7 +70,7 @@ DEBUG_GET_ONCE_LOG_OPTION(v4l2_log, "V4L2_LOG", U_LOGGING_WARN) DEBUG_GET_ONCE_NUM_OPTION(v4l2_exposure_absolute, "V4L2_EXPOSURE_ABSOLUTE", 10) -#define NUM_V4L2_BUFFERS 5 +#define NUM_V4L2_BUFFERS 32 /* @@ -123,6 +125,8 @@ struct v4l2_fs struct xrt_frame_node node; + struct u_sink_debug usd; + int fd; struct @@ -141,6 +145,7 @@ struct v4l2_fs } quirks; struct v4l2_frame frames[NUM_V4L2_BUFFERS]; + uint32_t used_frames; struct { @@ -167,7 +172,7 @@ struct v4l2_fs * Streaming thread entrypoint */ static void * -v4l2_fs_stream_run(void *ptr); +v4l2_fs_mainloop(void *ptr); /*! * Cast to derived type. @@ -207,6 +212,8 @@ v4l2_free_frame(struct xrt_frame *xf) struct v4l2_frame *vf = (struct v4l2_frame *)xf; struct v4l2_fs *vid = (struct v4l2_fs *)xf->owner; + vid->used_frames--; + if (!vid->is_running) { return; } @@ -658,7 +665,7 @@ v4l2_fs_stream_start(struct xrt_fs *xfs, vid->sink = xs; vid->is_running = true; vid->capture_type = capture_type; - if (pthread_create(&vid->stream_thread, NULL, v4l2_fs_stream_run, xfs)) { + if (pthread_create(&vid->stream_thread, NULL, v4l2_fs_mainloop, xfs)) { vid->is_running = false; V4L2_ERROR(vid, "error: Could not create thread"); return false; @@ -701,6 +708,7 @@ v4l2_fs_destroy(struct v4l2_fs *vid) // Stop the variable tracking. u_var_remove_root(vid); + u_sink_debug_destroy(&vid->usd); if (vid->descriptors != NULL) { free(vid->descriptors); @@ -781,14 +789,14 @@ v4l2_fs_create(struct xrt_frame_context *xfctx, xrt_frame_context_add(xfctx, &vid->node); // Start the variable tracking after we know what device we have. - // clang-format off + u_sink_debug_init(&vid->usd); u_var_add_root(vid, "V4L2 Frameserver", true); u_var_add_ro_text(vid, vid->base.name, "Card"); u_var_add_ro_u32(vid, &vid->ll, "Log Level"); for (size_t i = 0; i < vid->num_states; i++) { u_var_add_i32(vid, &vid->states[i].want[0].value, vid->states[i].name); } - // clang-format on + u_var_add_sink_debug(vid, &vid->usd, "Output"); v4l2_list_modes(vid); @@ -796,8 +804,10 @@ v4l2_fs_create(struct xrt_frame_context *xfctx, } void * -v4l2_fs_stream_run(void *ptr) +v4l2_fs_mainloop(void *ptr) { + SINK_TRACE_MARKER(); + struct xrt_fs *xfs = (struct xrt_fs *)ptr; struct v4l2_fs *vid = v4l2_fs(xfs); @@ -891,6 +901,10 @@ v4l2_fs_stream_run(void *ptr) v_buf.memory = v_bufrequest.memory; while (vid->is_running) { + if (vid->used_frames == NUM_V4L2_BUFFERS) { + V4L2_ERROR(vid, "No frames left"); + } + if (ioctl(vid->fd, VIDIOC_DQBUF, &v_buf) < 0) { V4L2_ERROR(vid, "error: Dequeue failed!"); vid->is_running = false; @@ -899,7 +913,10 @@ v4l2_fs_stream_run(void *ptr) v4l2_update_controls(vid); - V4L2_TRACE(vid, "Got frame #%u, index %i", v_buf.sequence, v_buf.index); + SINK_TRACE_IDENT(v4l2_fs_frame); + + V4L2_TRACE(vid, "Got frame #%u, index %i. Used frames are %i", v_buf.sequence, v_buf.index, + vid->used_frames); struct v4l2_frame *vf = &vid->frames[v_buf.index]; struct xrt_frame *xf = NULL; @@ -925,6 +942,11 @@ v4l2_fs_stream_run(void *ptr) vid->sink->push_frame(vid->sink, xf); + // Checks if active. + u_sink_debug_push_frame(&vid->usd, xf); + + vid->used_frames++; + // The frame is requeued as soon as the refcount reaches zero, // this can be done safely from another thread. xrt_frame_reference(&xf, NULL); diff --git a/src/xrt/drivers/vf/vf_driver.c b/src/xrt/drivers/vf/vf_driver.c index c5d9225c4..2b196002d 100644 --- a/src/xrt/drivers/vf/vf_driver.c +++ b/src/xrt/drivers/vf/vf_driver.c @@ -1,4 +1,4 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -12,22 +12,25 @@ #include "os/os_time.h" #include "os/os_threading.h" +#include "util/u_trace_marker.h" #include "util/u_var.h" #include "util/u_misc.h" #include "util/u_debug.h" #include "util/u_format.h" #include "util/u_frame.h" #include "util/u_logging.h" +#include "util/u_trace_marker.h" +#include "vf_interface.h" #include #include -#include "vf_interface.h" - +#include #include #include -#include +#include + /* * @@ -61,7 +64,6 @@ struct vf_fs struct os_thread_helper play_thread; - const char *path; GMainLoop *loop; GstElement *source; GstElement *testsink; @@ -91,6 +93,27 @@ struct vf_fs enum u_logging_level ll; }; +/*! + * Frame wrapping a GstSample/GstBuffer. + * + * @implements xrt_frame + */ +struct vf_frame +{ + struct xrt_frame base; + + GstSample *sample; + + GstVideoFrame frame; +}; + + +/* + * + * Cast helpers. + * + */ + /*! * Cast to derived type. */ @@ -100,6 +123,40 @@ vf_fs(struct xrt_fs *xfs) return (struct vf_fs *)xfs; } +/*! + * Cast to derived type. + */ +static inline struct vf_frame * +vf_frame(struct xrt_frame *xf) +{ + return (struct vf_frame *)xf; +} + + +/* + * + * Frame methods. + * + */ + +static void +vf_frame_destroy(struct xrt_frame *xf) +{ + SINK_TRACE_MARKER(); + + struct vf_frame *vff = vf_frame(xf); + + gst_video_frame_unmap(&vff->frame); + + if (vff->sample != NULL) { + gst_sample_unref(vff->sample); + vff->sample = NULL; + } + + free(vff); +} + + /* * * Misc helper functions @@ -107,9 +164,158 @@ vf_fs(struct xrt_fs *xfs) */ +static void +vf_fs_frame(struct vf_fs *vid, GstSample *sample) +{ + SINK_TRACE_MARKER(); + + // Noop. + if (!vid->sink) { + return; + } + + GstVideoInfo info; + GstBuffer *buffer; + GstCaps *caps; + buffer = gst_sample_get_buffer(sample); + caps = gst_sample_get_caps(sample); + + gst_video_info_init(&info); + gst_video_info_from_caps(&info, caps); + + static int seq = 0; + struct vf_frame *vff = U_TYPED_CALLOC(struct vf_frame); + + if (!gst_video_frame_map(&vff->frame, &info, buffer, GST_MAP_READ)) { + VF_ERROR(vid, "Failed to map frame %d", seq); + // Yes, we should do this here because we don't want the destroy function to run. + free(vff); + return; + } + + // We now want to hold onto the sample for as long as the frame lives. + gst_sample_ref(sample); + vff->sample = sample; + + // Hardcoded first plane. + int plane = 0; + + struct xrt_frame *xf = &vff->base; + xf->reference.count = 1; + xf->destroy = vf_frame_destroy; + xf->width = vid->width; + xf->height = vid->height; + xf->format = vid->format; + xf->stride = info.stride[plane]; + xf->data = vff->frame.data[plane]; + xf->stereo_format = vid->stereo_format; + xf->size = info.size; + xf->source_id = vid->base.source_id; + + //! @todo Proper sequence number and timestamp. + xf->source_sequence = seq; + xf->timestamp = os_monotonic_get_ns(); + + xrt_sink_push_frame(vid->sink, &vff->base); + + xrt_frame_reference(&xf, NULL); + vff = NULL; + + seq++; +} + +static GstFlowReturn +on_new_sample_from_sink(GstElement *elt, struct vf_fs *vid) +{ + SINK_TRACE_MARKER(); + GstSample *sample; + sample = gst_app_sink_pull_sample(GST_APP_SINK(elt)); + + if (!vid->got_sample) { + gint width; + gint height; + + GstCaps *caps = gst_sample_get_caps(sample); + GstStructure *structure = gst_caps_get_structure(caps, 0); + + gst_structure_get_int(structure, "width", &width); + gst_structure_get_int(structure, "height", &height); + + VF_DEBUG(vid, "video size is %dx%d", width, height); + vid->got_sample = true; + vid->width = width; + vid->height = height; + + // first sample is only used for getting metadata + return GST_FLOW_OK; + } + + // Takes ownership of the sample. + vf_fs_frame(vid, sample); + + // Done with sample now. + gst_sample_unref(sample); + + return GST_FLOW_OK; +} + +static void +print_gst_error(GstMessage *message) +{ + GError *err = NULL; + gchar *dbg_info = NULL; + + gst_message_parse_error(message, &err, &dbg_info); + U_LOG_E("ERROR from element %s: %s", GST_OBJECT_NAME(message->src), err->message); + U_LOG_E("Debugging info: %s", (dbg_info) ? dbg_info : "none"); + g_error_free(err); + g_free(dbg_info); +} + +static gboolean +on_source_message(GstBus *bus, GstMessage *message, struct vf_fs *vid) +{ + /* nil */ + switch (GST_MESSAGE_TYPE(message)) { + case GST_MESSAGE_EOS: + VF_DEBUG(vid, "Finished playback."); + g_main_loop_quit(vid->loop); + break; + case GST_MESSAGE_ERROR: + VF_ERROR(vid, "Received error."); + print_gst_error(message); + g_main_loop_quit(vid->loop); + break; + default: break; + } + return TRUE; +} + +static void * +vf_fs_mainloop(void *ptr) +{ + SINK_TRACE_MARKER(); + + struct vf_fs *vid = (struct vf_fs *)ptr; + + VF_DEBUG(vid, "Let's run!"); + g_main_loop_run(vid->loop); + VF_DEBUG(vid, "Going out!"); + + gst_object_unref(vid->testsink); + gst_element_set_state(vid->source, GST_STATE_NULL); + + + gst_object_unref(vid->source); + g_main_loop_unref(vid->loop); + + return NULL; +} + + /* * - * Exported functions. + * Frame server methods. * */ @@ -201,6 +407,13 @@ vf_fs_destroy(struct vf_fs *vid) free(vid); } + +/* + * + * Node methods. + * + */ + static void vf_fs_node_break_apart(struct xrt_frame_node *node) { @@ -215,188 +428,42 @@ vf_fs_node_destroy(struct xrt_frame_node *node) vf_fs_destroy(vid); } -#include -void -vf_fs_frame(struct vf_fs *vid, GstSample *sample) +/* + * + * Exported create functions and helper. + * + */ + +static struct xrt_fs * +alloc_and_init_common(struct xrt_frame_context *xfctx, // + enum xrt_format format, // + enum xrt_stereo_format stereo_format, // + gchar *pipeline_string) // { - GstBuffer *buffer; - buffer = gst_sample_get_buffer(sample); - GstCaps *caps = gst_sample_get_caps(sample); - - static int seq = 0; - - GstVideoFrame frame; - GstVideoInfo info; - gst_video_info_init(&info); - gst_video_info_from_caps(&info, caps); - if (gst_video_frame_map(&frame, &info, buffer, GST_MAP_READ)) { - - int plane = 0; - - struct xrt_frame *xf = NULL; - - u_frame_create_one_off(vid->format, vid->width, vid->height, &xf); - - //! @todo Sequence number and timestamp. - xf->width = vid->width; - xf->height = vid->height; - xf->format = vid->format; - xf->stereo_format = vid->stereo_format; - - xf->data = frame.data[plane]; - xf->stride = info.stride[plane]; - xf->size = info.size; - xf->source_id = vid->base.source_id; - xf->source_sequence = seq; - xf->timestamp = os_monotonic_get_ns(); - if (vid->sink) { - vid->sink->push_frame(vid->sink, xf); - // The frame is requeued as soon as the refcount reaches - // zero, this can be done safely from another thread. - // xrt_frame_reference(&xf, NULL); - } - gst_video_frame_unmap(&frame); - } else { - VF_ERROR(vid, "Failed to map frame %d", seq); - } - - seq++; -} - -static GstFlowReturn -on_new_sample_from_sink(GstElement *elt, struct vf_fs *vid) -{ - GstSample *sample; - sample = gst_app_sink_pull_sample(GST_APP_SINK(elt)); - - if (!vid->got_sample) { - gint width; - gint height; - - GstCaps *caps = gst_sample_get_caps(sample); - GstStructure *structure = gst_caps_get_structure(caps, 0); - - gst_structure_get_int(structure, "width", &width); - gst_structure_get_int(structure, "height", &height); - - VF_DEBUG(vid, "video size is %dx%d\n", width, height); - vid->got_sample = true; - vid->width = width; - vid->height = height; - - // first sample is only used for getting metadata - return GST_FLOW_OK; - } - - vf_fs_frame(vid, sample); - - gst_sample_unref(sample); - - return GST_FLOW_OK; -} - -static void -print_gst_error(GstMessage *message) -{ - GError *err = NULL; - gchar *dbg_info = NULL; - - gst_message_parse_error(message, &err, &dbg_info); - U_LOG_E("ERROR from element %s: %s\n", GST_OBJECT_NAME(message->src), err->message); - U_LOG_E("Debugging info: %s\n", (dbg_info) ? dbg_info : "none"); - g_error_free(err); - g_free(dbg_info); -} - -static gboolean -on_source_message(GstBus *bus, GstMessage *message, struct vf_fs *vid) -{ - /* nil */ - switch (GST_MESSAGE_TYPE(message)) { - case GST_MESSAGE_EOS: - VF_DEBUG(vid, "Finished playback\n"); - g_main_loop_quit(vid->loop); - break; - case GST_MESSAGE_ERROR: - VF_ERROR(vid, "Received error\n"); - print_gst_error(message); - g_main_loop_quit(vid->loop); - break; - default: break; - } - return TRUE; -} - -static void * -run_play_thread(void *ptr) -{ - struct vf_fs *vid = (struct vf_fs *)ptr; - - VF_DEBUG(vid, "Let's run!\n"); - g_main_loop_run(vid->loop); - VF_DEBUG(vid, "Going out\n"); - - gst_object_unref(vid->testsink); - gst_element_set_state(vid->source, GST_STATE_NULL); - - - gst_object_unref(vid->source); - g_main_loop_unref(vid->loop); - - return NULL; -} - -struct xrt_fs * -vf_fs_create(struct xrt_frame_context *xfctx, const char *path) -{ - if (path == NULL) { - U_LOG_E("No path given"); - return NULL; - } - - struct vf_fs *vid = U_TYPED_CALLOC(struct vf_fs); - vid->path = path; vid->got_sample = false; + vid->format = format; + vid->stereo_format = stereo_format; - gchar *loop = "false"; - - gchar *string = NULL; GstBus *bus = NULL; - - gst_init(0, NULL); - - if (!g_file_test(path, G_FILE_TEST_EXISTS)) { - VF_ERROR(vid, "File %s does not exist\n", path); + int ret = os_thread_helper_init(&vid->play_thread); + if (ret < 0) { + VF_ERROR(vid, "Failed to init thread"); + g_free(pipeline_string); + free(vid); return NULL; } vid->loop = g_main_loop_new(NULL, FALSE); + VF_DEBUG(vid, "Pipeline: %s", pipeline_string); -#if 0 - const gchar *caps = "video/x-raw,format=RGB"; - vid->format = XRT_FORMAT_R8G8B8; - vid->stereo_format = XRT_STEREO_FORMAT_SBS; -#endif - -#if 1 - const gchar *caps = "video/x-raw,format=YUY2"; - vid->format = XRT_FORMAT_YUYV422; - vid->stereo_format = XRT_STEREO_FORMAT_SBS; -#endif - - string = g_strdup_printf( - "multifilesrc location=\"%s\" loop=%s ! decodebin ! videoconvert ! " - "appsink caps=\"%s\" name=testsink", - path, loop, caps); - VF_DEBUG(vid, "Pipeline: %s\n", string); - vid->source = gst_parse_launch(string, NULL); - g_free(string); + vid->source = gst_parse_launch(pipeline_string, NULL); + g_free(pipeline_string); if (vid->source == NULL) { - VF_ERROR(vid, "Bad source\n"); + VF_ERROR(vid, "Bad source"); g_main_loop_unref(vid->loop); free(vid); return NULL; @@ -410,16 +477,21 @@ vf_fs_create(struct xrt_frame_context *xfctx, const char *path) gst_bus_add_watch(bus, (GstBusFunc)on_source_message, vid); gst_object_unref(bus); - int ret = os_thread_helper_start(&vid->play_thread, run_play_thread, vid); - if (!ret) { - VF_ERROR(vid, "Failed to start thread"); + ret = os_thread_helper_start(&vid->play_thread, vf_fs_mainloop, vid); + if (ret != 0) { + VF_ERROR(vid, "Failed to start thread '%i'", ret); + g_main_loop_unref(vid->loop); + free(vid); + return NULL; } - // we need one sample to determine frame size + // We need one sample to determine frame size. + VF_DEBUG(vid, "Waiting for frame"); gst_element_set_state(vid->source, GST_STATE_PLAYING); while (!vid->got_sample) { os_nanosleep(100 * 1000 * 1000); } + VF_DEBUG(vid, "Got first sample"); gst_element_set_state(vid->source, GST_STATE_PAUSED); vid->base.enumerate_modes = vf_fs_enumerate_modes; @@ -443,3 +515,69 @@ vf_fs_create(struct xrt_frame_context *xfctx, const char *path) return &(vid->base); } + +struct xrt_fs * +vf_fs_videotestsource(struct xrt_frame_context *xfctx, uint32_t width, uint32_t height) +{ + gst_init(0, NULL); + + enum xrt_format format = XRT_FORMAT_R8G8B8; + enum xrt_stereo_format stereo_format = XRT_STEREO_FORMAT_NONE; + + gchar *pipeline_string = g_strdup_printf( + "videotestsrc name=source ! " + "clockoverlay ! " + "videoconvert ! " + "videoscale ! " + "video/x-raw,format=RGB,width=%u,height=%u ! " + "appsink name=testsink", + width, height); + + return alloc_and_init_common(xfctx, format, stereo_format, pipeline_string); +} + +struct xrt_fs * +vf_fs_open_file(struct xrt_frame_context *xfctx, const char *path) +{ + if (path == NULL) { + U_LOG_E("No path given"); + return NULL; + } + + gst_init(0, NULL); + + if (!g_file_test(path, G_FILE_TEST_EXISTS)) { + U_LOG_E("File %s does not exist", path); + return NULL; + } + +#if 0 + const gchar *caps = "video/x-raw,format=RGB"; + enum xrt_format format = XRT_FORMAT_R8G8B8; + enum xrt_stereo_format stereo_format = XRT_STEREO_FORMAT_NONE; +#endif + +#if 0 + // For hand tracking + const gchar *caps = "video/x-raw,format=RGB"; + enum xrt_format format = XRT_FORMAT_R8G8B8; + enum xrt_stereo_format stereo_format = XRT_STEREO_FORMAT_SBS; +#endif + +#if 1 + const gchar *caps = "video/x-raw,format=YUY2"; + enum xrt_format format = XRT_FORMAT_YUYV422; + enum xrt_stereo_format stereo_format = XRT_STEREO_FORMAT_SBS; +#endif + + gchar *loop = "false"; + + gchar *pipeline_string = g_strdup_printf( + "multifilesrc location=\"%s\" loop=%s ! " + "decodebin ! " + "videoconvert ! " + "appsink caps=\"%s\" name=testsink", + path, loop, caps); + + return alloc_and_init_common(xfctx, format, stereo_format, pipeline_string); +} diff --git a/src/xrt/drivers/vf/vf_interface.h b/src/xrt/drivers/vf/vf_interface.h index 7919f83db..4f0ef1b2e 100644 --- a/src/xrt/drivers/vf/vf_interface.h +++ b/src/xrt/drivers/vf/vf_interface.h @@ -1,4 +1,4 @@ -// Copyright 2029, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -11,6 +11,7 @@ #include "xrt/xrt_frameserver.h" + #ifdef __cplusplus extern "C" { #endif @@ -23,14 +24,22 @@ extern "C" { */ /*! - * Create a vf frameserver + * Create a vf frameserver by opening a video file. * * @ingroup drv_vf */ struct xrt_fs * -vf_fs_create(struct xrt_frame_context *xfctx, const char *path); +vf_fs_open_file(struct xrt_frame_context *xfctx, const char *path); + +/*! + * Create a vf frameserver that uses the videotestsource. + * + * @ingroup drv_vf + */ +struct xrt_fs * +vf_fs_videotestsource(struct xrt_frame_context *xfctx, uint32_t width, uint32_t height); + #ifdef __cplusplus } - #endif diff --git a/src/xrt/drivers/vive/vive.h b/src/xrt/drivers/vive/vive.h index 7c6e65777..84f5196ab 100644 --- a/src/xrt/drivers/vive/vive.h +++ b/src/xrt/drivers/vive/vive.h @@ -16,8 +16,8 @@ * */ -#define VIVE_TRACE(d, ...) U_LOG_XDEV_IFL_T(&d->base, d->ll, __VA_ARGS__) -#define VIVE_DEBUG(d, ...) U_LOG_XDEV_IFL_D(&d->base, d->ll, __VA_ARGS__) -#define VIVE_INFO(d, ...) U_LOG_XDEV_IFL_I(&d->base, d->ll, __VA_ARGS__) -#define VIVE_WARN(d, ...) U_LOG_XDEV_IFL_W(&d->base, d->ll, __VA_ARGS__) -#define VIVE_ERROR(d, ...) U_LOG_XDEV_IFL_E(&d->base, d->ll, __VA_ARGS__) +#define VIVE_TRACE(d, ...) U_LOG_IFL_T(d->ll, __VA_ARGS__) +#define VIVE_DEBUG(d, ...) U_LOG_IFL_D(d->ll, __VA_ARGS__) +#define VIVE_INFO(d, ...) U_LOG_IFL_I(d->ll, __VA_ARGS__) +#define VIVE_WARN(d, ...) U_LOG_IFL_W(d->ll, __VA_ARGS__) +#define VIVE_ERROR(d, ...) U_LOG_IFL_E(d->ll, __VA_ARGS__) diff --git a/src/xrt/drivers/vive/vive_config.h b/src/xrt/drivers/vive/vive_config.h deleted file mode 100644 index aa7943601..000000000 --- a/src/xrt/drivers/vive/vive_config.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2020, Collabora, Ltd. -// SPDX-License-Identifier: BSL-1.0 -/*! - * @file - * @brief vive json header - * @author Lubosz Sarnecki - * @ingroup drv_vive - */ - -#pragma once - -#include - -struct vive_device; - -bool -vive_config_parse(struct vive_device *d, char *json_string); - - -struct vive_controller_device; - -bool -vive_config_parse_controller(struct vive_controller_device *d, char *json_string); diff --git a/src/xrt/drivers/vive/vive_controller.c b/src/xrt/drivers/vive/vive_controller.c index 4137215c2..dc4195bca 100644 --- a/src/xrt/drivers/vive/vive_controller.c +++ b/src/xrt/drivers/vive/vive_controller.c @@ -1,5 +1,5 @@ -// Copyright 2020, Collabora, Ltd. // Copyright 2016 Philipp Zabel +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -13,34 +13,39 @@ * originally written by Ryan Pavlik and available under the BSL-1.0. */ +#include "xrt/xrt_prober.h" + +#include "os/os_hid.h" +#include "os/os_threading.h" +#include "os/os_time.h" + +#include "math/m_api.h" +#include "math/m_predict.h" + +#include "util/u_json.h" +#include "util/u_misc.h" +#include "util/u_time.h" +#include "util/u_debug.h" +#include "util/u_device.h" +#include "util/u_trace_marker.h" + +#include "vive/vive_config.h" + +#include "vive.h" +#include "vive_protocol.h" +#include "vive_controller.h" #include #include #include #include -#include "xrt/xrt_prober.h" - -#include "math/m_api.h" -#include "util/u_debug.h" -#include "util/u_device.h" -#include "util/u_json.h" -#include "util/u_misc.h" -#include "util/u_time.h" -#include "os/os_hid.h" -#include "os/os_threading.h" -#include "os/os_time.h" - -#include "vive.h" -#include "vive_protocol.h" -#include "vive_controller.h" -#include "vive_config.h" - #ifdef XRT_OS_LINUX #include #include #endif + /* * * Defines & structs. @@ -106,6 +111,9 @@ vive_controller_device_destroy(struct xrt_device *xdev) os_thread_helper_destroy(&d->controller_thread); + // Now that the thread is not running we can destroy the lock. + os_mutex_destroy(&d->lock); + m_imu_3dof_close(&d->fusion); if (d->controller_hid) @@ -119,7 +127,8 @@ vive_controller_device_update_wand_inputs(struct xrt_device *xdev) { struct vive_controller_device *d = vive_controller_device(xdev); - os_thread_helper_lock(&d->controller_thread); + os_mutex_lock(&d->lock); + uint8_t buttons = d->state.buttons; /* @@ -169,15 +178,17 @@ vive_controller_device_update_wand_inputs(struct xrt_device *xdev) trigger_input->value.vec1.x = d->state.trigger; VIVE_TRACE(d, "Trigger: %f", d->state.trigger); - os_thread_helper_unlock(&d->controller_thread); + os_mutex_unlock(&d->lock); } static void vive_controller_device_update_index_inputs(struct xrt_device *xdev) { + XRT_TRACE_MARKER(); + struct vive_controller_device *d = vive_controller_device(xdev); - os_thread_helper_lock(&d->controller_thread); + os_mutex_lock(&d->lock); uint8_t buttons = d->state.buttons; /* @@ -285,7 +296,7 @@ vive_controller_device_update_index_inputs(struct xrt_device *xdev) VIVE_DEBUG(d, "Trackpad force: %f\n", (float)d->state.trackpad_force / UINT8_MAX); } - os_thread_helper_unlock(&d->controller_thread); + os_mutex_unlock(&d->lock); } @@ -298,9 +309,12 @@ _update_tracker_inputs(struct xrt_device *xdev) static void vive_controller_get_hand_tracking(struct xrt_device *xdev, enum xrt_input_name name, - uint64_t at_timestamp_ns, - struct xrt_hand_joint_set *out_value) + uint64_t requested_timestamp_ns, + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp_ns) { + XRT_TRACE_MARKER(); + struct vive_controller_device *d = vive_controller_device(xdev); if (name != XRT_INPUT_GENERIC_HAND_TRACKING_LEFT && name != XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT) { @@ -308,7 +322,7 @@ vive_controller_get_hand_tracking(struct xrt_device *xdev, return; } - enum xrt_hand hand = d->variant == CONTROLLER_INDEX_LEFT ? XRT_HAND_LEFT : XRT_HAND_RIGHT; + enum xrt_hand hand = d->config.variant == CONTROLLER_INDEX_LEFT ? XRT_HAND_LEFT : XRT_HAND_RIGHT; float thumb_curl = 0.0f; //! @todo place thumb preciely on the button that is touched/pressed @@ -325,7 +339,7 @@ vive_controller_get_hand_tracking(struct xrt_device *xdev, .index = (float)d->state.index_finger_trigger / UINT8_MAX, .thumb = thumb_curl}; - u_hand_joints_update_curl(&d->hand_tracking, hand, at_timestamp_ns, &values); + u_hand_joints_update_curl(&d->hand_tracking, hand, requested_timestamp_ns, &values); /* Because IMU is at the very -z end of the controller, the rotation @@ -348,6 +362,30 @@ vive_controller_get_hand_tracking(struct xrt_device *xdev, u_hand_joints_offset_valve_index_controller(hand, &static_offset, &hand_on_handle_pose); u_hand_joints_set_out_data(&d->hand_tracking, hand, &controller_relation, &hand_on_handle_pose, out_value); + + // This is a lie: currently, no pose-prediction or history is implemented for this driver. + *out_timestamp_ns = requested_timestamp_ns; + + out_value->is_active = true; +} + +static void +predict_pose(struct vive_controller_device *d, uint64_t at_timestamp_ns, struct xrt_space_relation *out_relation) +{ + timepoint_ns prediction_ns = at_timestamp_ns - d->imu.ts_received_ns; + double prediction_s = time_ns_to_s(prediction_ns); + + timepoint_ns monotonic_now_ns = os_monotonic_get_ns(); + timepoint_ns remaining_ns = at_timestamp_ns - monotonic_now_ns; + VIVE_TRACE(d, "dev %s At %ldns: Pose requested for +%ldns (%ldns), predicting %ldns", d->base.str, + monotonic_now_ns, remaining_ns, at_timestamp_ns, prediction_ns); + + //! @todo integrate position here + struct xrt_space_relation relation = {0}; + relation.pose.orientation = d->rot_filtered; + relation.relation_flags = XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT; + + m_predict_relation(&relation, prediction_s, out_relation); } static void @@ -368,22 +406,9 @@ vive_controller_device_get_tracked_pose(struct xrt_device *xdev, // Clear out the relation. U_ZERO(out_relation); - os_thread_helper_lock(&d->controller_thread); - - // Don't do anything if we have stopped. - if (!os_thread_helper_is_running_locked(&d->controller_thread)) { - os_thread_helper_unlock(&d->controller_thread); - return; - } - - out_relation->pose.orientation = d->rot_filtered; - - //! @todo assuming that orientation is actually currently tracked. - out_relation->relation_flags = (enum xrt_space_relation_flags)( - XRT_SPACE_RELATION_POSITION_VALID_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT | - XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT); - - os_thread_helper_unlock(&d->controller_thread); + os_mutex_lock(&d->lock); + predict_pose(d, at_timestamp_ns, out_relation); + os_mutex_unlock(&d->lock); struct xrt_vec3 pos = out_relation->pose.position; struct xrt_quat quat = out_relation->pose.orientation; @@ -462,7 +487,9 @@ vive_controller_device_set_output(struct xrt_device *xdev, enum xrt_output_name return; } + os_mutex_lock(&d->lock); vive_controller_haptic_pulse(d, value); + os_mutex_unlock(&d->lock); } static void @@ -526,11 +553,15 @@ cald_dt_ns(uint32_t dt_raw) static void vive_controller_handle_imu_sample(struct vive_controller_device *d, struct watchman_imu_sample *sample) { + XRT_TRACE_MARKER(); + /* ouvrt: "Time in 48 MHz ticks, but we are missing the low byte" */ uint32_t time_raw = d->last_ticks | (sample->timestamp_hi << 8); uint32_t dt_raw = calc_dt_raw_and_handle_overflow(d, time_raw); uint64_t dt_ns = cald_dt_ns(dt_raw); + d->imu.ts_received_ns = os_monotonic_get_ns(); + int16_t acc[3] = { __le16_to_cpu(sample->acc[0]), __le16_to_cpu(sample->acc[1]), @@ -543,18 +574,18 @@ vive_controller_handle_imu_sample(struct vive_controller_device *d, struct watch __le16_to_cpu(sample->gyro[2]), }; - float scale = (float)d->imu.acc_range / 32768.0f; + float scale = (float)d->config.imu.acc_range / 32768.0f; struct xrt_vec3 acceleration = { - scale * d->imu.acc_scale.x * acc[0] - d->imu.acc_bias.x, - scale * d->imu.acc_scale.y * acc[1] - d->imu.acc_bias.y, - scale * d->imu.acc_scale.z * acc[2] - d->imu.acc_bias.z, + scale * d->config.imu.acc_scale.x * acc[0] - d->config.imu.acc_bias.x, + scale * d->config.imu.acc_scale.y * acc[1] - d->config.imu.acc_bias.y, + scale * d->config.imu.acc_scale.z * acc[2] - d->config.imu.acc_bias.z, }; - scale = (float)d->imu.gyro_range / 32768.0f; + scale = (float)d->config.imu.gyro_range / 32768.0f; struct xrt_vec3 angular_velocity = { - scale * d->imu.gyro_scale.x * gyro[0] - d->imu.gyro_bias.x, - scale * d->imu.gyro_scale.y * gyro[1] - d->imu.gyro_bias.y, - scale * d->imu.gyro_scale.z * gyro[2] - d->imu.gyro_bias.z, + scale * d->config.imu.gyro_scale.x * gyro[0] - d->config.imu.gyro_bias.x, + scale * d->config.imu.gyro_scale.y * gyro[1] - d->config.imu.gyro_bias.y, + scale * d->config.imu.gyro_scale.z * gyro[2] - d->config.imu.gyro_bias.z, }; VIVE_TRACE(d, "ACC %f %f %f", acceleration.x, acceleration.y, acceleration.z); @@ -562,21 +593,21 @@ vive_controller_handle_imu_sample(struct vive_controller_device *d, struct watch /* */ - if (d->variant == CONTROLLER_VIVE_WAND) { + if (d->config.variant == CONTROLLER_VIVE_WAND) { struct xrt_vec3 fixed_acceleration = {.x = -acceleration.x, .y = -acceleration.z, .z = -acceleration.y}; acceleration = fixed_acceleration; struct xrt_vec3 fixed_angular_velocity = { .x = -angular_velocity.x, .y = -angular_velocity.z, .z = -angular_velocity.y}; angular_velocity = fixed_angular_velocity; - } else if (d->variant == CONTROLLER_INDEX_RIGHT) { + } else if (d->config.variant == CONTROLLER_INDEX_RIGHT) { struct xrt_vec3 fixed_acceleration = {.x = acceleration.z, .y = -acceleration.y, .z = acceleration.x}; acceleration = fixed_acceleration; struct xrt_vec3 fixed_angular_velocity = { .x = angular_velocity.z, .y = -angular_velocity.y, .z = angular_velocity.x}; angular_velocity = fixed_angular_velocity; - } else if (d->variant == CONTROLLER_INDEX_LEFT) { + } else if (d->config.variant == CONTROLLER_INDEX_LEFT) { struct xrt_vec3 fixed_acceleration = {.x = -acceleration.z, .y = acceleration.x, .z = -acceleration.y}; acceleration = fixed_acceleration; @@ -916,12 +947,16 @@ vive_controller_device_update(struct vive_controller_device *d) switch (buf[0]) { case VIVE_CONTROLLER_REPORT1_ID: + os_mutex_lock(&d->lock); vive_controller_decode_message(d, &((struct vive_controller_report1 *)buf)->message); + os_mutex_unlock(&d->lock); break; case VIVE_CONTROLLER_REPORT2_ID: + os_mutex_lock(&d->lock); vive_controller_decode_message(d, &((struct vive_controller_report2 *)buf)->message[0]); vive_controller_decode_message(d, &((struct vive_controller_report2 *)buf)->message[1]); + os_mutex_unlock(&d->lock); break; case VIVE_CONTROLLER_DISCONNECT_REPORT_ID: VIVE_DEBUG(d, "Controller disconnected."); break; default: VIVE_ERROR(d, "Unknown controller message type: %u", buf[0]); @@ -1023,29 +1058,29 @@ vive_controller_create(struct os_hid_device *controller_hid, enum watchman_gen w d->ll = debug_get_log_option_vive_log(); d->watchman_gen = WATCHMAN_GEN_UNKNOWN; - d->variant = CONTROLLER_UNKNOWN; + d->config.variant = CONTROLLER_UNKNOWN; d->watchman_gen = watchman_gen; m_imu_3dof_init(&d->fusion, M_IMU_3DOF_USE_GRAVITY_DUR_20MS); /* default values, will be queried from device */ - d->imu.gyro_range = 8.726646f; - d->imu.acc_range = 39.226600f; + d->config.imu.gyro_range = 8.726646f; + d->config.imu.acc_range = 39.226600f; - d->imu.acc_scale.x = 1.0f; - d->imu.acc_scale.y = 1.0f; - d->imu.acc_scale.z = 1.0f; - d->imu.gyro_scale.x = 1.0f; - d->imu.gyro_scale.y = 1.0f; - d->imu.gyro_scale.z = 1.0f; + d->config.imu.acc_scale.x = 1.0f; + d->config.imu.acc_scale.y = 1.0f; + d->config.imu.acc_scale.z = 1.0f; + d->config.imu.gyro_scale.x = 1.0f; + d->config.imu.gyro_scale.y = 1.0f; + d->config.imu.gyro_scale.z = 1.0f; - d->imu.acc_bias.x = 0.0f; - d->imu.acc_bias.y = 0.0f; - d->imu.acc_bias.z = 0.0f; - d->imu.gyro_bias.x = 0.0f; - d->imu.gyro_bias.y = 0.0f; - d->imu.gyro_bias.z = 0.0f; + d->config.imu.acc_bias.x = 0.0f; + d->config.imu.acc_bias.y = 0.0f; + d->config.imu.acc_bias.z = 0.0f; + d->config.imu.gyro_bias.x = 0.0f; + d->config.imu.gyro_bias.y = 0.0f; + d->config.imu.gyro_bias.z = 0.0f; d->controller_hid = controller_hid; @@ -1053,33 +1088,32 @@ vive_controller_create(struct os_hid_device *controller_hid, enum watchman_gen w d->base.get_tracked_pose = vive_controller_device_get_tracked_pose; d->base.set_output = vive_controller_device_set_output; - snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "%s %i", "Vive Controller", (int)(controller_num)); - - d->index = controller_num; - - //! @todo: reading range report fails for powered off controller - if (vive_get_imu_range_report(d->controller_hid, &d->imu.gyro_range, &d->imu.acc_range) != 0) { - VIVE_ERROR(d, "Could not get watchman IMU range packet!"); - free(d); - return 0; + if (vive_get_imu_range_report(d->controller_hid, &d->config.imu.gyro_range, &d->config.imu.acc_range) != 0) { + // reading range report fails for powered off controller + vive_controller_device_destroy(&d->base); + return NULL; } - VIVE_DEBUG(d, "Vive controller gyroscope range %f", d->imu.gyro_range); - VIVE_DEBUG(d, "Vive controller accelerometer range %f", d->imu.acc_range); + VIVE_DEBUG(d, "Vive controller gyroscope range %f", d->config.imu.gyro_range); + VIVE_DEBUG(d, "Vive controller accelerometer range %f", d->config.imu.acc_range); - // successful config parsing determines d->variant + // successful config parsing determines d->config.variant char *config = vive_read_config(d->controller_hid); + if (config != NULL) { - vive_config_parse_controller(d, config); + vive_config_parse_controller(&d->config, config, d->ll); free(config); } else { VIVE_ERROR(d, "Could not get Vive controller config\n"); - free(d); - return 0; + vive_controller_device_destroy(&d->base); + return NULL; } - if (d->variant == CONTROLLER_VIVE_WAND) { + snprintf(d->base.serial, XRT_DEVICE_NAME_LEN, "%s", d->config.firmware.device_serial_number); + + if (d->config.variant == CONTROLLER_VIVE_WAND) { d->base.name = XRT_DEVICE_VIVE_WAND; + snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "Vive Wand Controller (vive)"); SET_WAND_INPUT(SYSTEM_CLICK, SYSTEM_CLICK); SET_WAND_INPUT(SQUEEZE_CLICK, SQUEEZE_CLICK); @@ -1101,7 +1135,7 @@ vive_controller_create(struct os_hid_device *controller_hid, enum watchman_gen w d->base.num_binding_profiles = ARRAY_SIZE(binding_profiles_vive); d->base.device_type = XRT_DEVICE_TYPE_ANY_HAND_CONTROLLER; - } else if (d->variant == CONTROLLER_INDEX_LEFT || d->variant == CONTROLLER_INDEX_RIGHT) { + } else if (d->config.variant == CONTROLLER_INDEX_LEFT || d->config.variant == CONTROLLER_INDEX_RIGHT) { d->base.name = XRT_DEVICE_INDEX_CONTROLLER; SET_INDEX_INPUT(SYSTEM_CLICK, SYSTEM_CLICK); @@ -1132,28 +1166,32 @@ vive_controller_create(struct os_hid_device *controller_hid, enum watchman_gen w d->base.get_hand_tracking = vive_controller_get_hand_tracking; - enum xrt_hand hand = d->variant == CONTROLLER_INDEX_LEFT ? XRT_HAND_LEFT : XRT_HAND_RIGHT; + enum xrt_hand hand = d->config.variant == CONTROLLER_INDEX_LEFT ? XRT_HAND_LEFT : XRT_HAND_RIGHT; u_hand_joints_init_default_set(&d->hand_tracking, hand, XRT_HAND_TRACKING_MODEL_FINGERL_CURL, 1.0); d->base.binding_profiles = binding_profiles_index; d->base.num_binding_profiles = ARRAY_SIZE(binding_profiles_index); - if (d->variant == CONTROLLER_INDEX_LEFT) { + if (d->config.variant == CONTROLLER_INDEX_LEFT) { d->base.device_type = XRT_DEVICE_TYPE_LEFT_HAND_CONTROLLER; d->base.inputs[VIVE_CONTROLLER_HAND_TRACKING].name = XRT_INPUT_GENERIC_HAND_TRACKING_LEFT; - } else if (d->variant == CONTROLLER_INDEX_RIGHT) { + snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "Valve Index Left Controller (vive)"); + } else if (d->config.variant == CONTROLLER_INDEX_RIGHT) { d->base.device_type = XRT_DEVICE_TYPE_RIGHT_HAND_CONTROLLER; d->base.inputs[VIVE_CONTROLLER_HAND_TRACKING].name = XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT; + snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "Valve Index Right Controller (vive)"); } - } else if (d->variant == CONTROLLER_TRACKER_GEN1) { + } else if (d->config.variant == CONTROLLER_TRACKER_GEN1) { d->base.name = XRT_DEVICE_VIVE_TRACKER_GEN1; d->base.update_inputs = _update_tracker_inputs; d->base.device_type = XRT_DEVICE_TYPE_GENERIC_TRACKER; - } else if (d->variant == CONTROLLER_TRACKER_GEN2) { + snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "Vive Tracker Gen1 (vive)"); + } else if (d->config.variant == CONTROLLER_TRACKER_GEN2) { d->base.name = XRT_DEVICE_VIVE_TRACKER_GEN2; d->base.update_inputs = _update_tracker_inputs; d->base.device_type = XRT_DEVICE_TYPE_GENERIC_TRACKER; + snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "Vive Tracker Gen2 (vive)"); } else { d->base.name = XRT_DEVICE_GENERIC_HMD; d->base.device_type = XRT_DEVICE_TYPE_GENERIC_TRACKER; @@ -1161,18 +1199,27 @@ vive_controller_create(struct os_hid_device *controller_hid, enum watchman_gen w } if (d->controller_hid) { - int ret = os_thread_helper_start(&d->controller_thread, vive_controller_run_thread, d); + // Mutex before thread. + int ret = os_mutex_init(&d->lock); + if (ret != 0) { + VIVE_ERROR(d, "Failed to init mutex!"); + vive_controller_device_destroy(&d->base); + return NULL; + } + + ret = os_thread_helper_start(&d->controller_thread, vive_controller_run_thread, d); if (ret != 0) { VIVE_ERROR(d, "Failed to start mainboard thread!"); - vive_controller_device_destroy((struct xrt_device *)d); - return 0; + vive_controller_device_destroy(&d->base); + return NULL; } } VIVE_DEBUG(d, "Opened vive controller!\n"); d->base.orientation_tracking_supported = true; d->base.position_tracking_supported = false; - d->base.hand_tracking_supported = d->variant == CONTROLLER_INDEX_LEFT || d->variant == CONTROLLER_INDEX_RIGHT; + d->base.hand_tracking_supported = + d->config.variant == CONTROLLER_INDEX_LEFT || d->config.variant == CONTROLLER_INDEX_RIGHT; return d; } diff --git a/src/xrt/drivers/vive/vive_controller.h b/src/xrt/drivers/vive/vive_controller.h index 387998cd8..8c55dee68 100644 --- a/src/xrt/drivers/vive/vive_controller.h +++ b/src/xrt/drivers/vive/vive_controller.h @@ -17,8 +17,9 @@ #include "os/os_threading.h" #include "math/m_imu_3dof.h" #include "util/u_logging.h" - #include "util/u_hand_tracking.h" +#include "vive/vive_config.h" + #ifdef __cplusplus extern "C" { @@ -36,16 +37,6 @@ enum watchman_gen WATCHMAN_GEN_UNKNOWN }; -enum controller_variant -{ - CONTROLLER_VIVE_WAND, - CONTROLLER_INDEX_LEFT, - CONTROLLER_INDEX_RIGHT, - CONTROLLER_TRACKER_GEN1, - CONTROLLER_TRACKER_GEN2, - CONTROLLER_UNKNOWN -}; - /*! * A Vive Controller device, representing just a single controller. * @@ -58,20 +49,13 @@ struct vive_controller_device struct os_hid_device *controller_hid; struct os_thread_helper controller_thread; + struct os_mutex lock; struct { uint64_t time_ns; uint32_t last_sample_time_raw; - double acc_range; - double gyro_range; - struct xrt_vec3 acc_bias; - struct xrt_vec3 acc_scale; - struct xrt_vec3 gyro_bias; - struct xrt_vec3 gyro_scale; - - //! IMU position in tracking space. - struct xrt_pose trackref; + timepoint_ns ts_received_ns; } imu; struct m_imu_3dof fusion; @@ -113,22 +97,11 @@ struct vive_controller_device uint8_t battery; } state; - struct - { - uint32_t firmware_version; - uint8_t hardware_revision; - uint8_t hardware_version_micro; - uint8_t hardware_version_minor; - uint8_t hardware_version_major; - char mb_serial_number[32]; - char model_number[32]; - char device_serial_number[32]; - } firmware; - enum watchman_gen watchman_gen; - enum controller_variant variant; struct u_hand_tracking hand_tracking; + + struct vive_controller_config config; }; struct vive_controller_device * diff --git a/src/xrt/drivers/vive/vive_device.c b/src/xrt/drivers/vive/vive_device.c index 6da1c6964..e37c79e91 100644 --- a/src/xrt/drivers/vive/vive_device.c +++ b/src/xrt/drivers/vive/vive_device.c @@ -1,5 +1,5 @@ // Copyright 2016-2019, Philipp Zabel -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -17,8 +17,10 @@ #include "util/u_debug.h" #include "util/u_var.h" #include "util/u_time.h" +#include "util/u_trace_marker.h" #include "math/m_api.h" +#include "math/m_predict.h" #include "os/os_hid.h" #include "os/os_time.h" @@ -26,7 +28,6 @@ #include "vive.h" #include "vive_device.h" #include "vive_protocol.h" -#include "vive_config.h" #define VIVE_CLOCK_FREQ 48e6 // 48 MHz @@ -45,6 +46,8 @@ vive_device(struct xrt_device *xdev) static void vive_device_destroy(struct xrt_device *xdev) { + XRT_TRACE_MARKER(); + struct vive_device *d = vive_device(xdev); if (d->mainboard_dev) vive_mainboard_power_off(d); @@ -54,6 +57,8 @@ vive_device_destroy(struct xrt_device *xdev) os_thread_helper_destroy(&d->watchman_thread); os_thread_helper_destroy(&d->mainboard_thread); + // Now that the thread is not running we can destroy the lock. + m_imu_3dof_close(&d->fusion); if (d->mainboard_dev != NULL) { @@ -71,21 +76,21 @@ vive_device_destroy(struct xrt_device *xdev) d->watchman_dev = NULL; } - if (d->lh.sensors != NULL) { - free(d->lh.sensors); - d->lh.sensors = NULL; - d->lh.num_sensors = 0; - } + vive_config_teardown(&d->config); + + m_relation_history_destroy(&d->relation_hist); // Remove the variable tracking. u_var_remove_root(d); - free(d); + u_device_free(&d->base); } static void vive_device_update_inputs(struct xrt_device *xdev) { + XRT_TRACE_MARKER(); + struct vive_device *d = vive_device(xdev); VIVE_TRACE(d, "ENTER!"); } @@ -96,6 +101,8 @@ vive_device_get_tracked_pose(struct xrt_device *xdev, uint64_t at_timestamp_ns, struct xrt_space_relation *out_relation) { + XRT_TRACE_MARKER(); + struct vive_device *d = vive_device(xdev); if (name != XRT_INPUT_GENERIC_HEAD_POSE) { @@ -109,51 +116,25 @@ vive_device_get_tracked_pose(struct xrt_device *xdev, //! @todo Use this properly. (void)at_timestamp_ns; - os_thread_helper_lock(&d->sensors_thread); - - // Don't do anything if we have stopped. - if (!os_thread_helper_is_running_locked(&d->sensors_thread)) { - os_thread_helper_unlock(&d->sensors_thread); - return; - } - - out_relation->pose.orientation = d->rot_filtered; - - //! @todo assuming that orientation is actually currently tracked. - out_relation->relation_flags = (enum xrt_space_relation_flags)( - XRT_SPACE_RELATION_POSITION_VALID_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT | - XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT); - - os_thread_helper_unlock(&d->sensors_thread); + m_relation_history_get(d->relation_hist, out_relation, at_timestamp_ns); } static void vive_device_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { + XRT_TRACE_MARKER(); + + // Only supports two views. + assert(view_index < 2); + + u_device_get_view_pose(eye_relation, view_index, out_pose); + + // This is for the Index' canted displays, on the Vive [Pro] they are identity. struct vive_device *d = vive_device(xdev); - struct xrt_pose pose = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; - bool adjust = view_index == 0; - - pose.orientation = d->display.rot[view_index]; - pose.position.x = eye_relation->x / 2.0f; - pose.position.y = eye_relation->y / 2.0f; - pose.position.z = eye_relation->z / 2.0f; - - // Adjust for left/right while also making sure there aren't any -0.f. - if (pose.position.x > 0.0f && adjust) { - pose.position.x = -pose.position.x; - } - if (pose.position.y > 0.0f && adjust) { - pose.position.y = -pose.position.y; - } - if (pose.position.z > 0.0f && adjust) { - pose.position.z = -pose.position.z; - } - - *out_pose = pose; + out_pose->orientation = d->config.display.rot[view_index]; } static int @@ -178,11 +159,11 @@ vive_mainboard_get_device_info(struct vive_device *d) edid_vid = __be16_to_cpu(report.edid_vid); - d->firmware.display_firmware_version = __le32_to_cpu(report.display_firmware_version); + d->config.firmware.display_firmware_version = __le32_to_cpu(report.display_firmware_version); VIVE_INFO(d, "EDID Manufacturer ID: %c%c%c, Product code: 0x%04x", '@' + (edid_vid >> 10), '@' + ((edid_vid >> 5) & 0x1f), '@' + (edid_vid & 0x1f), __le16_to_cpu(report.edid_pid)); - VIVE_INFO(d, "Display firmware version: %u", d->firmware.display_firmware_version); + VIVE_INFO(d, "Display firmware version: %u", d->config.firmware.display_firmware_version); return 0; } @@ -237,7 +218,7 @@ vive_mainboard_decode_message(struct vive_device *d, struct vive_mainboard_statu if (d->board.button != report->button) { d->board.button = report->button; VIVE_TRACE(d, "Button %d.", report->button); - d->rot_filtered = (struct xrt_quat){0, 0, 0, 1}; + d->rot_filtered = (struct xrt_quat)XRT_QUAT_IDENTITY; } } @@ -279,9 +260,12 @@ cald_dt_ns(uint32_t dt_raw) static void update_imu(struct vive_device *d, const void *buffer) { + XRT_TRACE_MARKER(); + const struct vive_imu_report *report = buffer; const struct vive_imu_sample *sample = report->sample; uint8_t last_seq = d->imu.sequence; + d->imu.ts_received_ns = os_monotonic_get_ns(); int i, j; /* @@ -316,11 +300,11 @@ update_imu(struct vive_device *d, const void *buffer) (int16_t)__le16_to_cpu(sample->acc[2]), }; - scale = (float)d->imu.acc_range / 32768.0f; + scale = (float)d->config.imu.acc_range / 32768.0f; struct xrt_vec3 acceleration = { - scale * d->imu.acc_scale.x * acc[0] - d->imu.acc_bias.x, - scale * d->imu.acc_scale.y * acc[1] - d->imu.acc_bias.y, - scale * d->imu.acc_scale.z * acc[2] - d->imu.acc_bias.z, + scale * d->config.imu.acc_scale.x * acc[0] - d->config.imu.acc_bias.x, + scale * d->config.imu.acc_scale.y * acc[1] - d->config.imu.acc_bias.y, + scale * d->config.imu.acc_scale.z * acc[2] - d->config.imu.acc_bias.z, }; int16_t gyro[3] = { @@ -329,18 +313,18 @@ update_imu(struct vive_device *d, const void *buffer) (int16_t)__le16_to_cpu(sample->gyro[2]), }; - scale = (float)d->imu.gyro_range / 32768.0f; + scale = (float)d->config.imu.gyro_range / 32768.0f; struct xrt_vec3 angular_velocity = { - scale * d->imu.gyro_scale.x * gyro[0] - d->imu.gyro_bias.x, - scale * d->imu.gyro_scale.y * gyro[1] - d->imu.gyro_bias.y, - scale * d->imu.gyro_scale.z * gyro[2] - d->imu.gyro_bias.z, + scale * d->config.imu.gyro_scale.x * gyro[0] - d->config.imu.gyro_bias.x, + scale * d->config.imu.gyro_scale.y * gyro[1] - d->config.imu.gyro_bias.y, + scale * d->config.imu.gyro_scale.z * gyro[2] - d->config.imu.gyro_bias.z, }; VIVE_TRACE(d, "ACC %f %f %f", acceleration.x, acceleration.y, acceleration.z); VIVE_TRACE(d, "GYRO %f %f %f", angular_velocity.x, angular_velocity.y, angular_velocity.z); - switch (d->variant) { + switch (d->config.variant) { case VIVE_VARIANT_VIVE: // flip all except x axis acceleration.x = +acceleration.x; @@ -379,13 +363,25 @@ update_imu(struct vive_device *d, const void *buffer) } d->imu.time_ns += dt_ns; - d->last.acc = acceleration; - d->last.gyro = angular_velocity; d->imu.sequence = seq; m_imu_3dof_update(&d->fusion, d->imu.time_ns, &acceleration, &angular_velocity); d->rot_filtered = d->fusion.rot; + + struct xrt_space_relation rel = {0}; + rel.relation_flags = + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT; + rel.pose.orientation = d->rot_filtered; + + // Use d->imu.ts_received_ns instead of d->imu.time_ns. + // d->imu.time_ns is offset by an arbitrary value (I think so we get more floating-point precision in + // the 3dof fusion) - so it's not what we want in the global "time-space". + + // In contrast, d->imu.ts_received_ns is just "when we got the IMU timestamp" in the normal + // os_monotonic_get_ns() "time-space". Which is what we want. So we use it. + + m_relation_history_push(d->relation_hist, &rel, d->imu.ts_received_ns); } } @@ -483,6 +479,8 @@ _print_v1_pulse(struct vive_device *d, uint8_t sensor_id, uint32_t timestamp, ui static void _decode_pulse_report(struct vive_device *d, const void *buffer) { + XRT_TRACE_MARKER(); + const struct vive_headset_lighthouse_pulse_report *report = buffer; unsigned int i; @@ -505,8 +503,18 @@ _decode_pulse_report(struct vive_device *d, const void *buffer) continue; } + if (sensor_id == 0xfd) { + /* TODO: handle camera sync timestamp */ + continue; + } + + if (sensor_id == 0xfb) { + /* TODO: Only turns on when the camera is running but not every frame. */ + continue; + } + if (sensor_id > 31) { - VIVE_ERROR(d, "Unexpected sensor id: %04x\n", sensor_id); + VIVE_ERROR(d, "Unexpected sensor id: %04x", sensor_id); return; } @@ -564,7 +572,9 @@ vive_sensors_read_one_msg(struct vive_device *d, if (buffer[0] == report_id) { if (!_is_report_size_valid(d, ret, report_size, report_id)) return false; + process_cb(d, buffer); + } else { VIVE_ERROR(d, "Unexpected sensor report type %s (0x%x).", _sensors_get_report_string(buffer[0]), buffer[0]); @@ -599,6 +609,8 @@ _print_v2_pulse( static bool _print_pulse_report_v2(struct vive_device *d, const void *buffer) { + XRT_TRACE_MARKER(); + const struct vive_headset_lighthouse_v2_pulse_report *report = buffer; for (uint32_t i = 0; i < 4; i++) { @@ -717,41 +729,13 @@ vive_sensors_run_thread(void *ptr) return NULL; } -void -vive_init_defaults(struct vive_device *d) -{ - d->display.eye_target_width_in_pixels = 1080; - d->display.eye_target_height_in_pixels = 1200; - - d->display.rot[0].w = 1.0f; - d->display.rot[1].w = 1.0f; - - d->imu.gyro_range = 8.726646f; - d->imu.acc_range = 39.226600f; - - d->imu.acc_scale.x = 1.0f; - d->imu.acc_scale.y = 1.0f; - d->imu.acc_scale.z = 1.0f; - - d->imu.gyro_scale.x = 1.0f; - d->imu.gyro_scale.y = 1.0f; - d->imu.gyro_scale.z = 1.0f; - - d->rot_filtered.w = 1.0f; - - for (int view = 0; view < 2; view++) { - d->distortion[view].aspect_x_over_y = 0.89999997615814209f; - d->distortion[view].grow_for_undistort = 0.5f; - d->distortion[view].undistort_r2_cutoff = 1.0f; - } -} - - static bool compute_distortion(struct xrt_device *xdev, int view, float u, float v, struct xrt_uv_triplet *result) { + XRT_TRACE_MARKER(); + struct vive_device *d = vive_device(xdev); - return u_compute_distortion_vive(&d->distortion[view], u, v, result); + return u_compute_distortion_vive(&d->config.distortion[view], u, v, result); } struct vive_device * @@ -760,11 +744,18 @@ vive_device_create(struct os_hid_device *mainboard_dev, struct os_hid_device *watchman_dev, enum VIVE_VARIANT variant) { + XRT_TRACE_MARKER(); + enum u_device_alloc_flags flags = (enum u_device_alloc_flags)(U_DEVICE_ALLOC_HMD | U_DEVICE_ALLOC_TRACKING_NONE); struct vive_device *d = U_DEVICE_ALLOCATE(struct vive_device, flags, 1, 0); - d->base.hmd->blend_mode = XRT_BLEND_MODE_OPAQUE; + m_relation_history_create(&d->relation_hist); + + size_t idx = 0; + d->base.hmd->blend_modes[idx++] = XRT_BLEND_MODE_OPAQUE; + d->base.hmd->num_blend_modes = idx; + d->base.update_inputs = vive_device_update_inputs; d->base.get_tracked_pose = vive_device_get_tracked_pose; d->base.get_view_pose = vive_device_get_view_pose; @@ -775,28 +766,18 @@ vive_device_create(struct os_hid_device *mainboard_dev, d->sensors_dev = sensors_dev; d->ll = debug_get_log_option_vive_log(); d->watchman_dev = watchman_dev; - d->variant = variant; d->base.hmd->distortion.models = XRT_DISTORTION_MODEL_COMPUTE; d->base.hmd->distortion.preferred = XRT_DISTORTION_MODEL_COMPUTE; d->base.compute_distortion = compute_distortion; - vive_init_defaults(d); - - switch (variant) { - case VIVE_VARIANT_VIVE: snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "HTC Vive"); break; - case VIVE_VARIANT_PRO: snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "HTC Vive Pro"); break; - case VIVE_VARIANT_INDEX: snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "Valve Index"); break; - default: snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "Unknown Vive device"); - } - if (d->mainboard_dev) { vive_mainboard_power_on(d); vive_mainboard_get_device_info(d); } - vive_read_firmware(d->sensors_dev, &d->firmware.firmware_version, &d->firmware.hardware_revision, - &d->firmware.hardware_version_micro, &d->firmware.hardware_version_minor, - &d->firmware.hardware_version_major); + vive_read_firmware(d->sensors_dev, &d->config.firmware.firmware_version, &d->config.firmware.hardware_revision, + &d->config.firmware.hardware_version_micro, &d->config.firmware.hardware_version_minor, + &d->config.firmware.hardware_version_major); /* VIVE_INFO(d, "Firmware version %u %s@%s FPGA %u.%u", @@ -804,18 +785,21 @@ vive_device_create(struct os_hid_device *mainboard_dev, report.fpga_version_major, report.fpga_version_minor); */ - VIVE_INFO(d, "Firmware version %u", d->firmware.firmware_version); - VIVE_INFO(d, "Hardware revision: %d rev %d.%d.%d", d->firmware.hardware_revision, - d->firmware.hardware_version_major, d->firmware.hardware_version_minor, - d->firmware.hardware_version_micro); + VIVE_INFO(d, "Firmware version %u", d->config.firmware.firmware_version); + VIVE_INFO(d, "Hardware revision: %d rev %d.%d.%d", d->config.firmware.hardware_revision, + d->config.firmware.hardware_version_major, d->config.firmware.hardware_version_minor, + d->config.firmware.hardware_version_micro); - vive_get_imu_range_report(d->sensors_dev, &d->imu.gyro_range, &d->imu.acc_range); - VIVE_INFO(d, "Vive gyroscope range %f", d->imu.gyro_range); - VIVE_INFO(d, "Vive accelerometer range %f", d->imu.acc_range); + vive_get_imu_range_report(d->sensors_dev, &d->config.imu.gyro_range, &d->config.imu.acc_range); + VIVE_INFO(d, "Vive gyroscope range %f", d->config.imu.gyro_range); + VIVE_INFO(d, "Vive accelerometer range %f", d->config.imu.acc_range); char *config = vive_read_config(d->sensors_dev); + + d->config.ll = d->ll; + // usb connected HMD variant is known because of USB id, config parsing relies on it. if (config != NULL) { - vive_config_parse(d, config); + vive_config_parse(&d->config, config, d->ll); free(config); } @@ -825,14 +809,14 @@ vive_device_create(struct os_hid_device *mainboard_dev, double lens_horizontal_separation = 0.057863; double eye_to_screen_distance = 0.023226876441867737; - uint32_t w_pixels = d->display.eye_target_width_in_pixels; - uint32_t h_pixels = d->display.eye_target_height_in_pixels; + uint32_t w_pixels = d->config.display.eye_target_width_in_pixels; + uint32_t h_pixels = d->config.display.eye_target_height_in_pixels; // Main display. d->base.hmd->screens[0].w_pixels = (int)w_pixels * 2; d->base.hmd->screens[0].h_pixels = (int)h_pixels; - if (d->variant == VIVE_VARIANT_INDEX) { + if (d->config.variant == VIVE_VARIANT_INDEX) { lens_horizontal_separation = 0.06; h_meters = 0.07; // eye relief knob adjusts this around [0.0255(near)-0.275(far)] @@ -881,14 +865,13 @@ vive_device_create(struct os_hid_device *mainboard_dev, m_imu_3dof_init(&d->fusion, M_IMU_3DOF_USE_GRAVITY_DUR_20MS); u_var_add_root(d, "Vive Device", true); + u_var_add_gui_header(d, &d->gui.fusion, "3DoF Fusion"); + m_imu_3dof_add_vars(&d->fusion, d, ""); u_var_add_gui_header(d, &d->gui.calibration, "Calibration"); - u_var_add_vec3_f32(d, &d->imu.acc_scale, "acc_scale"); - u_var_add_vec3_f32(d, &d->imu.acc_bias, "acc_bias"); - u_var_add_vec3_f32(d, &d->imu.gyro_scale, "gyro_scale"); - u_var_add_vec3_f32(d, &d->imu.gyro_bias, "gyro_bias"); - u_var_add_gui_header(d, &d->gui.last, "Last data"); - u_var_add_vec3_f32(d, &d->last.acc, "acc"); - u_var_add_vec3_f32(d, &d->last.gyro, "gyro"); + u_var_add_vec3_f32(d, &d->config.imu.acc_scale, "acc_scale"); + u_var_add_vec3_f32(d, &d->config.imu.acc_bias, "acc_bias"); + u_var_add_vec3_f32(d, &d->config.imu.gyro_scale, "gyro_scale"); + u_var_add_vec3_f32(d, &d->config.imu.gyro_bias, "gyro_bias"); int ret; @@ -915,6 +898,14 @@ vive_device_create(struct os_hid_device *mainboard_dev, d->base.position_tracking_supported = false; d->base.device_type = XRT_DEVICE_TYPE_HMD; + switch (d->config.variant) { + case VIVE_VARIANT_VIVE: snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "HTC Vive (vive)"); break; + case VIVE_VARIANT_PRO: snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "HTC Vive Pro (vive)"); break; + case VIVE_VARIANT_INDEX: snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "Valve Index (vive)"); break; + case VIVE_UNKNOWN: snprintf(d->base.str, XRT_DEVICE_NAME_LEN, "Unknown HMD (vive)"); break; + } + snprintf(d->base.serial, XRT_DEVICE_NAME_LEN, "%s", d->config.firmware.device_serial_number); + ret = os_thread_helper_start(&d->sensors_thread, vive_sensors_run_thread, d); if (ret != 0) { VIVE_ERROR(d, "Failed to start sensors thread!"); diff --git a/src/xrt/drivers/vive/vive_device.h b/src/xrt/drivers/vive/vive_device.h index eef5121be..9f7f2d67e 100644 --- a/src/xrt/drivers/vive/vive_device.h +++ b/src/xrt/drivers/vive/vive_device.h @@ -14,6 +14,9 @@ #include "os/os_threading.h" #include "util/u_logging.h" #include "util/u_distortion_mesh.h" +#include "math/m_relation_history.h" + +#include "vive/vive_config.h" #include "vive_lighthouse.h" @@ -21,37 +24,6 @@ extern "C" { #endif - -/*! - * A lighthouse consisting of sensors. - * - * All sensors are placed in IMU space. - */ -struct lh_model -{ - struct lh_sensor *sensors; - size_t num_sensors; -}; - -/*! - * A single lighthouse senosor point and normal, in IMU space. - */ -struct lh_sensor -{ - struct xrt_vec3 pos; - uint32_t _pad0; - struct xrt_vec3 normal; - uint32_t _pad1; -}; - -enum VIVE_VARIANT -{ - VIVE_UNKNOWN = 0, - VIVE_VARIANT_VIVE, - VIVE_VARIANT_PRO, - VIVE_VARIANT_INDEX -}; - /*! * @implements xrt_device */ @@ -64,53 +36,20 @@ struct vive_device struct lighthouse_watchman watchman; - enum VIVE_VARIANT variant; - struct os_thread_helper sensors_thread; struct os_thread_helper watchman_thread; struct os_thread_helper mainboard_thread; - struct lh_model lh; - struct { uint64_t time_ns; uint8_t sequence; uint32_t last_sample_time_raw; - double acc_range; - double gyro_range; - struct xrt_vec3 acc_bias; - struct xrt_vec3 acc_scale; - struct xrt_vec3 gyro_bias; - struct xrt_vec3 gyro_scale; - - //! IMU position in tracking space. - struct xrt_pose trackref; + timepoint_ns ts_received_ns; } imu; struct m_imu_3dof fusion; - struct - { - struct xrt_vec3 acc; - struct xrt_vec3 gyro; - } last; - - struct - { - double lens_separation; - double persistence; - int eye_target_height_in_pixels; - int eye_target_width_in_pixels; - - struct xrt_quat rot[2]; - - //! Head position in tracking space. - struct xrt_pose trackref; - //! Head position in IMU space. - struct xrt_pose imuref; - } display; - struct { uint16_t ipd; @@ -119,20 +58,8 @@ struct vive_device uint8_t button; } board; - struct - { - uint32_t display_firmware_version; - uint32_t firmware_version; - uint8_t hardware_revision; - uint8_t hardware_version_micro; - uint8_t hardware_version_minor; - uint8_t hardware_version_major; - char mb_serial_number[32]; - char model_number[32]; - char device_serial_number[32]; - } firmware; - struct xrt_quat rot_filtered; + struct m_relation_history *relation_hist; enum u_logging_level ll; bool disconnect_notified; @@ -140,10 +67,10 @@ struct vive_device struct { bool calibration; - bool last; + bool fusion; } gui; - struct u_vive_values distortion[2]; + struct vive_config config; }; struct vive_device * diff --git a/src/xrt/drivers/vive/vive_prober.c b/src/xrt/drivers/vive/vive_prober.c index a01919b05..29233c23a 100644 --- a/src/xrt/drivers/vive/vive_prober.c +++ b/src/xrt/drivers/vive/vive_prober.c @@ -11,11 +11,16 @@ #include "util/u_debug.h" +#include "util/u_trace_marker.h" #include "vive_device.h" #include "vive_controller.h" #include "vive_prober.h" +#include "../ht/ht_interface.h" +#include "../multi_wrapper/multi.h" +#include "xrt/xrt_config_drivers.h" + static const char VIVE_PRODUCT_STRING[] = "HTC Vive"; static const char VIVE_PRO_PRODUCT_STRING[] = "VIVE Pro"; static const char VALVE_INDEX_PRODUCT_STRING[] = "Index HMD"; @@ -24,6 +29,10 @@ static const char VIVE_MANUFACTURER_STRING[] = "HTC"; DEBUG_GET_ONCE_LOG_OPTION(vive_log, "VIVE_LOG", U_LOGGING_WARN) +#ifdef XRT_BUILD_DRIVER_HANDTRACKING +DEBUG_GET_ONCE_BOOL_OPTION(vive_use_handtracking, "VIVE_USE_HANDTRACKING", false) +#endif + static int log_vive_string(struct xrt_prober *xp, struct xrt_prober_device *dev, enum xrt_prober_string type) { @@ -129,6 +138,8 @@ init_vive_pro(struct xrt_prober *xp, enum u_logging_level ll, struct xrt_device **out_xdev) { + XRT_TRACE_MARKER(); + log_vive_device(ll, xp, dev); if (!xrt_prober_match_string(xp, dev, XRT_PROBER_STRING_MANUFACTURER, VIVE_MANUFACTURER_STRING) || @@ -199,8 +210,10 @@ init_valve_index(struct xrt_prober *xp, struct xrt_prober_device **devices, size_t num_devices, enum u_logging_level ll, - struct xrt_device **out_xdev) + struct xrt_device **out_xdevs) { + XRT_TRACE_MARKER(); + log_vive_device(ll, xp, dev); if (!xrt_prober_match_string(xp, dev, XRT_PROBER_STRING_MANUFACTURER, VALVE_INDEX_MANUFACTURER_STRING) || @@ -239,9 +252,32 @@ init_valve_index(struct xrt_prober *xp, return -1; } - *out_xdev = &d->base; + int out_idx = 0; - return 1; + out_xdevs[out_idx++] = &d->base; + +#ifdef XRT_BUILD_DRIVER_HANDTRACKING + if (debug_get_bool_option_vive_use_handtracking()) { + struct t_stereo_camera_calibration *cal = NULL; + + struct xrt_pose head_in_left_cam; + // vive_get_stereo_camera_calibration(&ss->hmd->hmd.config, &cal, &head_in_left_cam); + vive_get_stereo_camera_calibration(&d->config, &cal, &head_in_left_cam); + + struct xrt_device *ht = ht_device_create(xp, cal); + if (ht != NULL) { // Returns NULL if there's a problem and the hand tracker can't start. By no means a + // fatal error. + struct xrt_device *wrap = + multi_create_tracking_override(XRT_TRACKING_OVERRIDE_ATTACHED, ht, &d->base, + XRT_INPUT_GENERIC_HEAD_POSE, &head_in_left_cam); + out_xdevs[out_idx++] = wrap; + } + // Don't need it anymore. And it's not even created unless we hit this codepath, which is somewhat hard. + t_stereo_camera_calibration_reference(&cal, NULL); + } +#endif + + return out_idx; } int @@ -252,6 +288,8 @@ vive_found(struct xrt_prober *xp, cJSON *attached_data, struct xrt_device **out_xdev) { + XRT_TRACE_MARKER(); + struct xrt_prober_device *dev = devices[index]; enum u_logging_level ll = debug_get_log_option_vive_log(); @@ -281,6 +319,8 @@ vive_controller_found(struct xrt_prober *xp, cJSON *attached_data, struct xrt_device **out_xdevs) { + XRT_TRACE_MARKER(); + struct xrt_prober_device *dev = devices[index]; int ret; diff --git a/src/xrt/drivers/vive/vive_protocol.c b/src/xrt/drivers/vive/vive_protocol.c index e11ddfb5e..4fa60cdea 100644 --- a/src/xrt/drivers/vive/vive_protocol.c +++ b/src/xrt/drivers/vive/vive_protocol.c @@ -10,17 +10,19 @@ */ #include "math/m_mathinclude.h" - -#include -#include #include "math/m_api.h" -#include "vive_protocol.h" - #include "util/u_debug.h" #include "util/u_misc.h" #include "util/u_json.h" #include "util/u_logging.h" +#include "util/u_trace_marker.h" + +#include "vive_protocol.h" + +#include +#include + const struct vive_headset_power_report power_on_report = { .id = VIVE_HEADSET_POWER_REPORT_ID, @@ -64,13 +66,16 @@ const struct vive_headset_power_report power_off_report = { char * vive_read_config(struct os_hid_device *hid_dev) { + XRT_TRACE_MARKER(); + struct vive_config_start_report start_report = { .id = VIVE_CONFIG_START_REPORT_ID, }; int ret = os_hid_get_feature_timeout(hid_dev, &start_report, sizeof(start_report), 100); if (ret < 0) { - U_LOG_E("Could not get config start report."); + // e.g. watchman receiver has no connected device (controller powered off) + U_LOG_I("Could not get config start report for device, connected device may be powered off (%d).", ret); return NULL; } @@ -151,7 +156,7 @@ vive_get_imu_range_report(struct os_hid_device *hid_dev, double *gyro_range, dou ret = os_hid_get_feature_timeout(hid_dev, &report, sizeof(report), 100); if (ret < 0) { - U_LOG_E("Could not get range report!"); + U_LOG_I("Could not get range report, connected device may be powered off (%d)!", ret); return ret; } diff --git a/src/xrt/drivers/wmr/wmr_common.h b/src/xrt/drivers/wmr/wmr_common.h new file mode 100644 index 000000000..29849cbbf --- /dev/null +++ b/src/xrt/drivers/wmr/wmr_common.h @@ -0,0 +1,47 @@ +// Copyright 2020-2021, N Madsen. +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Defines and constants related to WMR driver code. + * @author nima01 + * @author Jakob Bornecrantz + * @ingroup drv_wmr + */ + +#pragma once + + +#ifdef __cplusplus +extern "C" { +#endif + + +/*! + * Defines for the WMR driver. + * + * @ingroup drv_wmr + * @{ + */ + +#define MS_HOLOLENS_MANUFACTURER_STRING "Microsoft" +#define MS_HOLOLENS_PRODUCT_STRING "HoloLens Sensors" + +#define MICROSOFT_VID 0x045e +#define HOLOLENS_SENSORS_PID 0x0659 + +#define HP_VID 0x03f0 +#define REVERB_G1_PID 0x0c6a +#define REVERB_G2_PID 0x0580 + +#define LENOVO_VID 0x17ef +#define EXPLORER_PID 0xb801 + +/*! + * @} + */ + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/wmr/wmr_config.c b/src/xrt/drivers/wmr/wmr_config.c new file mode 100644 index 000000000..d274037f1 --- /dev/null +++ b/src/xrt/drivers/wmr/wmr_config.c @@ -0,0 +1,275 @@ +/* Copyright 2021, Jan Schmidt + * SPDX-License-Identifier: BSL-1.0 + */ +/*! + * @file + * @brief Driver code to read WMR config blocks + * @author Jan Schmidt + * @ingroup drv_wmr + */ +#include +#include "math/m_api.h" + +#include "util/u_misc.h" +#include "util/u_json.h" + +#include "wmr_config.h" + +#define WMR_TRACE(ll, ...) U_LOG_IFL_T(ll, __VA_ARGS__) +#define WMR_DEBUG(ll, ...) U_LOG_IFL_D(ll, __VA_ARGS__) +#define WMR_INFO(ll, ...) U_LOG_IFL_I(ll, __VA_ARGS__) +#define WMR_WARN(ll, ...) U_LOG_IFL_W(ll, __VA_ARGS__) +#define WMR_ERROR(ll, ...) U_LOG_IFL_E(ll, __VA_ARGS__) + +#define JSON_INT(a, b, c) u_json_get_int(u_json_get(a, b), c) +#define JSON_FLOAT(a, b, c) u_json_get_float(u_json_get(a, b), c) +#define JSON_DOUBLE(a, b, c) u_json_get_double(u_json_get(a, b), c) +#define JSON_VEC3(a, b, c) u_json_get_vec3_array(u_json_get(a, b), c) +#define JSON_MATRIX_3X3(a, b, c) u_json_get_matrix_3x3(u_json_get(a, b), c) +#define JSON_STRING(a, b, c) u_json_get_string_into_array(u_json_get(a, b), c, sizeof(c)) + +static void +wmr_config_init_defaults(struct wmr_hmd_config *c) +{ + memset(c, 0, sizeof(struct wmr_hmd_config)); + + // initialize default sensor transforms + math_pose_identity(&c->eye_params[0].pose); + math_pose_identity(&c->eye_params[1].pose); + math_pose_identity(&c->accel_pose); + math_pose_identity(&c->gyro_pose); + math_pose_identity(&c->mag_pose); +} + +static void +wmr_config_compute_pose(struct xrt_pose *out_pose, const struct xrt_vec3 *tx, const struct xrt_matrix_3x3 *rx) +{ + // Adjust the coordinate system / conventions of the raw Tx and Rx config to yield a usable xrt_pose + // The config stores a 3x3 rotation matrix and a vec3 translation. + // Translation is applied after rotation, and the coordinate system is flipped in YZ. + + struct xrt_matrix_3x3 coordsys = {.v = {1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, -1.0}}; + + struct xrt_matrix_3x3 rx_adj; + math_matrix_3x3_multiply(&coordsys, rx, &rx_adj); + math_quat_from_matrix_3x3(&rx_adj, &out_pose->orientation); + + struct xrt_vec3 v; + math_matrix_3x3_transform_vec3(&coordsys, tx, &v); + math_matrix_3x3_transform_vec3(&rx_adj, &v, &out_pose->position); +} + +static bool +wmr_config_parse_display(struct wmr_hmd_config *c, cJSON *display, enum u_logging_level ll) +{ + cJSON *json_eye = cJSON_GetObjectItem(display, "AssignedEye"); + char *json_eye_name = cJSON_GetStringValue(json_eye); + + if (json_eye_name == NULL) { + WMR_ERROR(ll, "Invalid/missing eye assignment block"); + return false; + } + + struct wmr_distortion_eye_config *eye = NULL; + if (!strcmp(json_eye_name, "CALIBRATION_DisplayEyeLeft")) { + eye = &c->eye_params[0]; + } else if (!strcmp(json_eye_name, "CALIBRATION_DisplayEyeRight")) { + eye = &c->eye_params[1]; + } else { + WMR_ERROR(ll, "Unknown AssignedEye \"%s\"", json_eye_name); + return false; + } + + /* Extract display panel parameters */ + cJSON *affine = cJSON_GetObjectItem(display, "Affine"); + if (affine == NULL || u_json_get_float_array(affine, eye->affine_xform.v, 9) != 9) { + WMR_ERROR(ll, "Missing affine transform for AssignedEye \"%s\"", json_eye_name); + return false; + } + + if (!JSON_FLOAT(display, "DisplayWidth", &eye->display_size.x) || + !JSON_FLOAT(display, "DisplayHeight", &eye->display_size.y)) + return false; + + cJSON *visible_area_center = cJSON_GetObjectItem(display, "VisibleAreaCenter"); + if (visible_area_center == NULL || !JSON_FLOAT(visible_area_center, "X", &eye->visible_center.x) || + !JSON_FLOAT(visible_area_center, "Y", &eye->visible_center.y)) { + return false; + } + + if (!JSON_DOUBLE(display, "VisibleAreaRadius", &eye->visible_radius)) + return false; + + /* Compute eye pose */ + cJSON *rt = cJSON_GetObjectItem(display, "Rt"); + cJSON *rx = cJSON_GetObjectItem(rt, "Rotation"); + if (rt == NULL || rx == NULL) + return false; + + struct xrt_vec3 translation; + struct xrt_matrix_3x3 rotation; + + if (!JSON_VEC3(rt, "Translation", &translation)) + return false; + + if (u_json_get_float_array(rx, rotation.v, 9) != 9) + return false; + + wmr_config_compute_pose(&eye->pose, &translation, &rotation); + + /* Parse color distortion channels */ + const char *channel_names[] = {"DistortionRed", "DistortionGreen", "DistortionBlue"}; + + for (int channel = 0; channel < 3; ++channel) { + struct wmr_distortion_3K *distortion3K = &eye->distortion3K[channel]; + + cJSON *dist = cJSON_GetObjectItemCaseSensitive(display, channel_names[channel]); + if (!dist) { + WMR_ERROR(ll, "Missing distortion channel info %s", channel_names[channel]); + return false; + } + + const char *model_type = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(dist, "ModelType")); + if (model_type == NULL) { + WMR_ERROR(ll, "Missing distortion type"); + return false; + } + + if (!strcmp(model_type, "CALIBRATION_DisplayDistortionModelPolynomial3K")) { + distortion3K->model = WMR_DISTORTION_MODEL_POLYNOMIAL_3K; + } else { + distortion3K->model = WMR_DISTORTION_MODEL_UNKNOWN; + WMR_ERROR(ll, "Unknown distortion model %s", model_type); + return false; + } + + int param_count; + double parameters[5]; + + if (!JSON_INT(dist, "ModelParameterCount", ¶m_count)) { + WMR_ERROR(ll, "Missing distortion parameters"); + return false; + } + + cJSON *params_json = cJSON_GetObjectItemCaseSensitive(dist, "ModelParameters"); + if (params_json == NULL || + u_json_get_double_array(params_json, parameters, param_count) != (size_t)param_count) { + WMR_ERROR(ll, "Missing distortion parameters"); + return false; + } + + distortion3K->eye_center.x = parameters[0]; + distortion3K->eye_center.y = parameters[1]; + + distortion3K->k[0] = parameters[2]; + distortion3K->k[1] = parameters[3]; + distortion3K->k[2] = parameters[4]; + } + + return true; +} + +static bool +wmr_config_parse_inertial_sensor(struct wmr_hmd_config *c, cJSON *sensor, enum u_logging_level ll) +{ + struct xrt_pose *out_pose; + + const char *sensor_type = cJSON_GetStringValue(cJSON_GetObjectItem(sensor, "SensorType")); + if (sensor_type == NULL) { + WMR_WARN(ll, "Missing sensor type"); + return false; + } + + if (!strcmp(sensor_type, "CALIBRATION_InertialSensorType_Gyro")) { + out_pose = &c->gyro_pose; + } else if (!strcmp(sensor_type, "CALIBRATION_InertialSensorType_Accelerometer")) { + out_pose = &c->accel_pose; + } else if (!strcmp(sensor_type, "CALIBRATION_InertialSensorType_Magnetometer")) { + out_pose = &c->mag_pose; + } else { + WMR_WARN(ll, "Unhandled sensor type \"%s\"", sensor_type); + return false; + } + + struct xrt_vec3 translation; + struct xrt_matrix_3x3 rotation; + + cJSON *rt = cJSON_GetObjectItem(sensor, "Rt"); + cJSON *rx = cJSON_GetObjectItem(rt, "Rotation"); + if (rt == NULL || rx == NULL) { + WMR_WARN(ll, "Missing Inertial Sensor calibration"); + return false; + } + + if (!JSON_VEC3(rt, "Translation", &translation) || u_json_get_float_array(rx, rotation.v, 9) != 9) { + WMR_WARN(ll, "Invalid Inertial Sensor calibration"); + return false; + } + + wmr_config_compute_pose(out_pose, &translation, &rotation); + + return true; +} + + +static bool +wmr_config_parse_calibration(struct wmr_hmd_config *c, cJSON *calib_info, enum u_logging_level ll) +{ + cJSON *item = NULL; + + // calib_info is object with keys "Cameras", "Displays", and "InertialSensors" + cJSON *displays = cJSON_GetObjectItemCaseSensitive(calib_info, "Displays"); + if (!cJSON_IsArray(displays)) { + WMR_ERROR(ll, "Displays: not found or not an Array"); + return false; + } + + cJSON_ArrayForEach(item, displays) + { + if (!wmr_config_parse_display(c, item, ll)) { + WMR_ERROR(ll, "Error parsing Display entry"); + return false; + } + } + + cJSON *sensors = cJSON_GetObjectItemCaseSensitive(calib_info, "InertialSensors"); + if (!cJSON_IsArray(sensors)) { + WMR_ERROR(ll, "InertialSensors: not found or not an Array"); + return false; + } + + cJSON_ArrayForEach(item, sensors) + { + if (!wmr_config_parse_inertial_sensor(c, item, ll)) { + WMR_WARN(ll, "Error parsing InertialSensor entry"); + } + } + + return true; +} + + +bool +wmr_config_parse(struct wmr_hmd_config *c, char *json_string, enum u_logging_level ll) +{ + wmr_config_init_defaults(c); + + cJSON *json_root = cJSON_Parse(json_string); + if (!cJSON_IsObject(json_root)) { + WMR_ERROR(ll, "Could not parse JSON data."); + cJSON_Delete(json_root); + return false; + } + + cJSON *calib_info = cJSON_GetObjectItemCaseSensitive(json_root, "CalibrationInformation"); + if (!cJSON_IsObject(calib_info)) { + WMR_ERROR(ll, "CalibrationInformation object not found"); + cJSON_Delete(json_root); + return false; + } + + bool res = wmr_config_parse_calibration(c, calib_info, ll); + + cJSON_Delete(json_root); + return res; +} diff --git a/src/xrt/drivers/wmr/wmr_config.h b/src/xrt/drivers/wmr/wmr_config.h new file mode 100644 index 000000000..f1279b39c --- /dev/null +++ b/src/xrt/drivers/wmr/wmr_config.h @@ -0,0 +1,71 @@ +/* Copyright 2021 Jan Schmidt + * SPDX-License-Identifier: BSL-1.0 + */ +/*! + * @file + * @brief WMR and MS HoloLens configuration structures + * @author Jan Schmidt + * @ingroup drv_wmr + */ + +#pragma once + +#include "math/m_vec2.h" +#include "util/u_logging.h" + +enum wmr_distortion_model +{ + WMR_DISTORTION_MODEL_UNKNOWN = 0, + WMR_DISTORTION_MODEL_POLYNOMIAL_3K +}; + +#ifdef __cplusplus +extern "C" { +#endif + +struct wmr_distortion_3K +{ + enum wmr_distortion_model model; + + /* X/Y center of the distortion (pixels) */ + struct xrt_vec2 eye_center; + /* k1,k2,k3 params for radial distortion as + * per the radial distortion model in + * https://docs.opencv.org/master/d9/d0c/group__calib3d.html */ + double k[3]; +}; + +struct wmr_distortion_eye_config +{ + /* 3x3 camera matrix that moves from normalised camera coords (X/Z & Y/Z) to undistorted pixels */ + struct xrt_matrix_3x3 affine_xform; + /* Eye pose in world space */ + struct xrt_pose pose; + /* Radius of the (undistorted) visible area from the center (pixels) (I think) */ + double visible_radius; + + /* Width, Height (pixels) of the full display */ + struct xrt_vec2 display_size; + /* Center for the eye viewport visibility (pixels) */ + struct xrt_vec2 visible_center; + + /* RGB distortion params */ + struct wmr_distortion_3K distortion3K[3]; +}; + +struct wmr_hmd_config +{ + /* Left and Right eye mapping and distortion params */ + struct wmr_distortion_eye_config eye_params[2]; + + struct xrt_pose accel_pose; + struct xrt_pose gyro_pose; + struct xrt_pose mag_pose; +}; + +bool +wmr_config_parse(struct wmr_hmd_config *c, char *json_string, enum u_logging_level ll); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/wmr/wmr_config_key.h b/src/xrt/drivers/wmr/wmr_config_key.h new file mode 100644 index 000000000..b35852510 --- /dev/null +++ b/src/xrt/drivers/wmr/wmr_config_key.h @@ -0,0 +1,78 @@ +/* Copyright 2021 Jan Schmidt + * SPDX-License-Identifier: BSL-1.0 + */ + +/* + * This block is XOR-ed with the configuration stored in all WMR headsets seen so far. + * It was derived from a handful of raw config blocks read from headsets using + * https://github.com/pH5/wmr-config + */ + +// clang-format off +const uint8_t wmr_config_key[] = +{ + 0x2F, 0xC8, 0x0F, 0x38, 0xDD, 0x00, 0xF6, 0x5C, 0xA1, 0x31, 0xEF, 0xF1, 0xEA, 0x6F, 0xA0, 0xF8, + 0x26, 0xB5, 0x9B, 0x39, 0xCF, 0x3A, 0x88, 0xC8, 0x2E, 0x17, 0xC0, 0x63, 0x5B, 0x46, 0x27, 0xBB, + 0x98, 0x2F, 0x0E, 0x2A, 0x90, 0x4B, 0x28, 0x2D, 0x82, 0x76, 0xE5, 0x28, 0x72, 0x50, 0x8A, 0xF0, + 0xBF, 0x84, 0x54, 0x3B, 0xA8, 0x77, 0x91, 0xCE, 0x87, 0x80, 0x53, 0x2F, 0x07, 0xAD, 0x1B, 0x3F, + 0x8C, 0x67, 0x33, 0x2E, 0xEB, 0x6A, 0x2A, 0x52, 0x77, 0x7C, 0x1F, 0x02, 0x11, 0x9E, 0x2A, 0x59, + 0x5C, 0x94, 0x0E, 0x4F, 0xF5, 0x44, 0x54, 0x01, 0xE7, 0x8F, 0x66, 0xF0, 0xAD, 0x68, 0x71, 0x3C, + 0x6D, 0x2E, 0x1C, 0xE3, 0x11, 0x46, 0xF7, 0x7F, 0x02, 0x6C, 0x15, 0xA0, 0x10, 0xEE, 0x3B, 0x14, + 0xAE, 0x6C, 0xA7, 0x3F, 0xAF, 0x83, 0x6A, 0xD7, 0x12, 0x88, 0x53, 0xFE, 0xEB, 0x5C, 0x78, 0x85, + 0xAF, 0x1F, 0x80, 0x7F, 0xB6, 0xDA, 0x7C, 0x0E, 0x84, 0xB5, 0x02, 0x8E, 0x92, 0xA3, 0x5B, 0x83, + 0x56, 0x11, 0x7B, 0xDF, 0x80, 0xB3, 0x4C, 0x13, 0x8E, 0x61, 0x61, 0xE6, 0x82, 0x8C, 0xDA, 0x08, + 0x76, 0x88, 0xBF, 0x85, 0x7F, 0xE4, 0x28, 0x26, 0x1F, 0xB5, 0x67, 0x80, 0x63, 0xD9, 0x26, 0xD4, + 0x91, 0xD1, 0xC1, 0x51, 0xCE, 0x61, 0x64, 0x2B, 0x56, 0xAE, 0x3D, 0x06, 0x5D, 0xCD, 0xF7, 0x05, + 0x9A, 0x6F, 0xEB, 0x2E, 0xC2, 0x69, 0x42, 0x86, 0xBE, 0x78, 0x32, 0xC3, 0xA8, 0x94, 0xA3, 0x97, + 0x84, 0x07, 0xF1, 0x6E, 0x3F, 0x10, 0xDE, 0x2B, 0xB1, 0x41, 0x1A, 0x59, 0xE3, 0x7F, 0x23, 0xDE, + 0xF3, 0x13, 0x23, 0xD1, 0x60, 0x1C, 0xBB, 0xC5, 0x4A, 0xB1, 0xC6, 0x02, 0x38, 0x7B, 0xFE, 0xEF, + 0xB6, 0x50, 0x16, 0x23, 0x4B, 0xD4, 0xEF, 0xEA, 0x67, 0xEC, 0x44, 0x2C, 0xC0, 0xA6, 0x2F, 0x0D, + 0x6E, 0x17, 0x52, 0xC8, 0x26, 0xCB, 0x63, 0x85, 0x72, 0x8D, 0xBA, 0xD8, 0x09, 0xA3, 0x89, 0x64, + 0x70, 0x12, 0xC6, 0xDF, 0x4C, 0x28, 0xD4, 0xB8, 0x49, 0x18, 0x69, 0x22, 0x36, 0xF1, 0x00, 0xC3, + 0x91, 0xB3, 0x7C, 0xA0, 0xAA, 0x7D, 0x9E, 0x27, 0x65, 0xCD, 0x16, 0x3C, 0x71, 0x3C, 0xCC, 0xF5, + 0x02, 0xD1, 0xA5, 0x06, 0x95, 0xB2, 0x1E, 0x71, 0x92, 0x6F, 0xC2, 0xD2, 0xEF, 0x58, 0x7B, 0xD0, + 0x53, 0x5E, 0xE9, 0xB6, 0xCA, 0x1C, 0x13, 0x96, 0xAC, 0xF1, 0xF5, 0x19, 0xD9, 0x8A, 0x1D, 0xA9, + 0x0D, 0xAE, 0xD0, 0xF4, 0xB3, 0xDD, 0x2D, 0x43, 0x6E, 0x41, 0x22, 0xDC, 0x09, 0xA4, 0x92, 0x42, + 0xC5, 0x32, 0x7D, 0xD7, 0xA2, 0x57, 0xA5, 0xA5, 0x11, 0xD2, 0x22, 0xD0, 0xD7, 0x75, 0xFF, 0xFC, + 0x2F, 0x66, 0xB5, 0xA9, 0xA3, 0x2B, 0xB4, 0x2B, 0x14, 0xEF, 0xC4, 0xB4, 0x18, 0xF0, 0x56, 0x55, + 0x93, 0x6A, 0x30, 0xF6, 0xDF, 0x14, 0x23, 0xF7, 0x2A, 0xDC, 0x4C, 0xE7, 0x78, 0x2B, 0x66, 0x22, + 0xB4, 0xAC, 0x2E, 0x03, 0xF2, 0xEF, 0xEC, 0x35, 0xAF, 0x22, 0x6D, 0x05, 0x12, 0x6D, 0xC5, 0x2C, + 0x12, 0x9B, 0x7D, 0x66, 0x47, 0x8F, 0xC0, 0x81, 0xCD, 0xFE, 0x97, 0x4C, 0x01, 0xC1, 0xD7, 0x24, + 0xD8, 0x46, 0xFD, 0x29, 0xF7, 0xE1, 0x61, 0xF3, 0xA0, 0x69, 0xBD, 0x23, 0x35, 0x7E, 0x66, 0xB1, + 0xF6, 0x3A, 0xD9, 0xF8, 0x29, 0xA2, 0x99, 0x8E, 0xF7, 0xBC, 0xCC, 0xD6, 0x37, 0x11, 0x09, 0xC8, + 0x07, 0x68, 0x5D, 0xF6, 0xC2, 0x73, 0xE8, 0xE5, 0x2D, 0xA4, 0x62, 0x0E, 0x9D, 0xC2, 0x76, 0xA9, + 0x94, 0x06, 0x19, 0x39, 0x9F, 0x8F, 0x83, 0x62, 0xE9, 0xDE, 0xED, 0x76, 0xFD, 0xBC, 0x42, 0x77, + 0x1E, 0x49, 0x18, 0xD8, 0x15, 0x22, 0x55, 0x5C, 0xD3, 0xF2, 0x7F, 0xD0, 0x8A, 0x27, 0x82, 0x05, + 0xD3, 0xBD, 0x27, 0x0C, 0x2F, 0xB9, 0x72, 0xE9, 0x9D, 0x6B, 0xD3, 0xD6, 0xD6, 0x84, 0xA4, 0x1F, + 0x6C, 0x26, 0x7C, 0x61, 0xE0, 0x7E, 0x58, 0x05, 0xAD, 0xC5, 0xE1, 0x14, 0x3A, 0xD7, 0x40, 0x6A, + 0x52, 0x57, 0x82, 0xAA, 0x9B, 0xF0, 0xCA, 0x60, 0x5D, 0x6C, 0xC0, 0xA4, 0x6B, 0xF3, 0x87, 0x6D, + 0x04, 0x80, 0x2C, 0x7B, 0xEB, 0x9F, 0xD4, 0x03, 0x81, 0x91, 0xD0, 0xB9, 0x74, 0xAE, 0x19, 0xBF, + 0x48, 0x63, 0x8F, 0x8C, 0xEE, 0xBC, 0xB4, 0xC0, 0x16, 0x4A, 0xF5, 0x5E, 0x1C, 0x7A, 0xDB, 0xD5, + 0xA4, 0x16, 0x92, 0xCB, 0x52, 0x86, 0xCB, 0xD1, 0x1E, 0x1D, 0xEE, 0x90, 0x01, 0x90, 0x52, 0x52, + 0x52, 0x8C, 0x25, 0x0A, 0xB7, 0xDE, 0x10, 0x51, 0xB8, 0x23, 0x5C, 0xCB, 0x32, 0x6A, 0xB0, 0xB9, + 0xA4, 0x58, 0xB6, 0x14, 0x28, 0xF0, 0xFB, 0xC2, 0xCD, 0x6F, 0x5E, 0x10, 0x48, 0xAD, 0x1F, 0xC8, + 0xCE, 0x4F, 0x09, 0xDA, 0xF8, 0xD0, 0x84, 0x44, 0x8C, 0x57, 0x4B, 0xE1, 0x87, 0x5B, 0x79, 0xD0, + 0x93, 0x38, 0x57, 0x65, 0x31, 0x55, 0xF2, 0xD6, 0x1F, 0x6C, 0xC9, 0xD1, 0x3A, 0x17, 0x3C, 0x4F, + 0x97, 0x23, 0x07, 0xB9, 0xB6, 0xB5, 0x32, 0x28, 0x24, 0x0E, 0xCC, 0x1A, 0xA1, 0x74, 0x39, 0x06, + 0xD9, 0x52, 0xD6, 0x38, 0xFC, 0x95, 0xBF, 0x84, 0x3A, 0x76, 0xA3, 0xC3, 0x54, 0xF2, 0x71, 0x4D, + 0x2D, 0xE8, 0x9F, 0x58, 0x19, 0xE9, 0xD3, 0x5A, 0xCE, 0x30, 0x1E, 0xB5, 0xEE, 0xB5, 0x83, 0xF4, + 0xB9, 0x23, 0xF3, 0xA1, 0xFC, 0xEA, 0x68, 0x2F, 0xAF, 0x22, 0x73, 0xF2, 0x21, 0x66, 0x8C, 0x29, + 0xF2, 0x34, 0x7A, 0x39, 0xB9, 0x3C, 0x2C, 0x96, 0x54, 0x7A, 0x7E, 0xA5, 0x24, 0x98, 0xF7, 0x06, + 0x78, 0x28, 0x70, 0x7A, 0x3C, 0x73, 0x8D, 0x82, 0xB1, 0x9C, 0x1E, 0xD9, 0xDB, 0xBB, 0xEF, 0x3F, + 0xC2, 0x0F, 0xAF, 0x73, 0x0E, 0xC0, 0x01, 0x2E, 0x5B, 0x8A, 0xC4, 0x39, 0x5A, 0x71, 0xA9, 0x2B, + 0xD9, 0xD3, 0x9A, 0x0D, 0x28, 0x95, 0xFE, 0x7E, 0xD8, 0xC7, 0x73, 0xDD, 0x77, 0x52, 0x56, 0x94, + 0x80, 0x93, 0xD1, 0xFF, 0x02, 0x28, 0xE0, 0x18, 0xA1, 0xF2, 0x7E, 0x9A, 0x1C, 0xF2, 0x7B, 0x76, + 0x2C, 0xF0, 0xB7, 0x39, 0xF3, 0x10, 0x08, 0x90, 0x8F, 0xA6, 0xEB, 0x5F, 0xF5, 0x1A, 0xB1, 0x72, + 0xF0, 0x1B, 0x7A, 0xF4, 0xF7, 0x4D, 0x5C, 0xC0, 0x82, 0x1F, 0x27, 0xCE, 0xA4, 0x52, 0xB2, 0xE8, + 0x24, 0xC7, 0xCA, 0x8C, 0xB9, 0xCB, 0x6C, 0xC5, 0xA0, 0x42, 0x18, 0x7F, 0xE5, 0xFA, 0xA9, 0x8E, + 0xA0, 0xF4, 0x58, 0x78, 0xB9, 0x30, 0x86, 0x49, 0x01, 0x15, 0x8E, 0xB0, 0x22, 0x8C, 0xF5, 0x12, + 0x64, 0xE6, 0x69, 0x90, 0xD6, 0x86, 0x92, 0x9B, 0x83, 0xD4, 0xF7, 0x01, 0x15, 0x9A, 0x7C, 0xF8, + 0xB3, 0xCD, 0x0A, 0xA1, 0x3D, 0x49, 0x90, 0x21, 0x69, 0xD7, 0x25, 0xFC, 0x1A, 0x64, 0x22, 0x77, + 0x7A, 0xBF, 0x3C, 0x1C, 0x4B, 0x06, 0x6E, 0x83, 0x03, 0x5D, 0x5C, 0x76, 0xEA, 0x84, 0x29, 0xB5, + 0x7C, 0xC0, 0x74, 0xBC, 0x4A, 0x21, 0x7B, 0xDC, 0xFE, 0x1B, 0x1F, 0x77, 0x64, 0x20, 0x59, 0x6A, + 0x0B, 0x48, 0xC2, 0x0E, 0x2D, 0xFF, 0xCE, 0x4C, 0x06, 0xED, 0x0E, 0x1C, 0xB6, 0x1A, 0x62, 0x79, + 0xEC, 0x25, 0xD6, 0x89, 0xBF, 0x4F, 0x16, 0x75, 0x82, 0xD7, 0x98, 0x5C, 0xBA, 0x75, 0xBA, 0xD3, + 0x2D, 0xC7, 0x47, 0xF3, 0xB6, 0x31, 0x54, 0xE0, 0x86, 0xFE, 0x29, 0x8E, 0xE2, 0x92, 0x79, 0x89, + 0xE4, 0x43, 0xB4, 0x9C, 0xF7, 0xED, 0x1B, 0xA6, 0x0B, 0x0C, 0x69, 0x23, 0xF4, 0x7D, 0x0A, 0xA2, + 0x8C, 0xEC, 0xD5, 0x2C, 0x8E, 0xB6, 0x20, 0x8F, 0xA6, 0xB9, 0x86, 0xB8, 0xBA, 0x59, 0xA3, 0xA7 +}; diff --git a/src/xrt/drivers/wmr/wmr_hmd.c b/src/xrt/drivers/wmr/wmr_hmd.c new file mode 100644 index 000000000..8208b36bd --- /dev/null +++ b/src/xrt/drivers/wmr/wmr_hmd.c @@ -0,0 +1,962 @@ +// Copyright 2018, Philipp Zabel. +// Copyright 2020-2021, N Madsen. +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Driver code for a WMR HMD. + * @author Philipp Zabel + * @author nima01 + * @author Jakob Bornecrantz + * @ingroup drv_wmr + */ + +#include "xrt/xrt_config_os.h" +#include "xrt/xrt_device.h" + +#include "os/os_time.h" +#include "os/os_hid.h" + +#include "math/m_mathinclude.h" +#include "math/m_api.h" +#include "math/m_vec2.h" +#include "math/m_predict.h" + +#include "util/u_var.h" +#include "util/u_misc.h" +#include "util/u_time.h" +#include "util/u_debug.h" +#include "util/u_device.h" +#include "util/u_distortion_mesh.h" + +#include "wmr_hmd.h" +#include "wmr_common.h" +#include "wmr_config_key.h" +#include "wmr_protocol.h" + +#include +#include +#include +#include +#ifndef XRT_OS_WINDOWS +#include // for sleep() +#endif + +static int +wmr_hmd_activate_reverb(struct wmr_hmd *wh); +static void +wmr_hmd_deactivate_reverb(struct wmr_hmd *wh); + +const struct wmr_headset_descriptor headset_map[] = { + {WMR_HEADSET_GENERIC, NULL, "Unknown WMR HMD", NULL, NULL}, /* Catch-all for unknown headsets */ + {WMR_HEADSET_REVERB_G1, "HP Reverb VR Headset VR1000-2xxx", "HP Reverb", wmr_hmd_activate_reverb, + wmr_hmd_deactivate_reverb}, + {WMR_HEADSET_REVERB_G2, "HP Reverb Virtual Reality Headset G2", "HP Reverb G2", wmr_hmd_activate_reverb, + wmr_hmd_deactivate_reverb}, + {WMR_HEADSET_SAMSUNG_800ZAA, "Samsung Windows Mixed Reality 800ZAA", "Samsung Odyssey", NULL, NULL}, + {WMR_HEADSET_LENOVO_EXPLORER, "Lenovo VR-2511N", "Lenovo Explorer", NULL, NULL}, +}; +const int headset_map_n = sizeof(headset_map) / sizeof(headset_map[0]); + +/* + * + * Hololens packets. + * + */ + +static void +hololens_unknown_17_decode_packet(struct wmr_hmd *wh, const unsigned char *buffer, int size) +{ + if (size >= 7) { + WMR_TRACE(wh, "Got packet 0x17 (%i)\n\t%02x %02x %02x %02x %02x %02x %02x ", size, buffer[0], buffer[1], + buffer[2], buffer[3], buffer[4], buffer[5], buffer[6]); + } else { + WMR_TRACE(wh, "Got packet 0x17 (%i)", size); + } +} + +static void +hololens_unknown_05_06_0E_decode_packet(struct wmr_hmd *wh, const unsigned char *buffer, int size) +{ + if (size >= 45) { + WMR_TRACE(wh, + "Got controller (%i)\n\t%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x | %02x %02x %02x " + "%02x %02x %02x %02x %02x %02x %02x | %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x", + size, buffer[0], buffer[1], buffer[2], buffer[3], buffer[4], buffer[5], buffer[6], buffer[7], + buffer[8], buffer[9], buffer[10], buffer[11], buffer[12], buffer[13], buffer[14], buffer[15], + buffer[16], buffer[17], buffer[18], buffer[19], buffer[20], buffer[21], buffer[22], + buffer[23], buffer[24], buffer[25], buffer[26], buffer[27], buffer[28], buffer[29]); + } else { + WMR_TRACE(wh, "Got controller packet (%i)\n\t%02x", size, buffer[0]); + } +} + +static void +hololens_sensors_decode_packet(struct wmr_hmd *wh, + struct hololens_sensors_packet *pkt, + const unsigned char *buffer, + int size) +{ + WMR_TRACE(wh, " "); + + if (size != 497 && size != 381) { + WMR_ERROR(wh, "invalid hololens sensor packet size (expected 381 or 497 but got %d)", size); + return; + } + + pkt->id = read8(&buffer); + for (int i = 0; i < 4; i++) { + pkt->temperature[i] = read16(&buffer); + } + + for (int i = 0; i < 4; i++) { + pkt->gyro_timestamp[i] = read64(&buffer); + } + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 32; j++) { + pkt->gyro[i][j] = read16(&buffer); + } + } + + for (int i = 0; i < 4; i++) { + pkt->accel_timestamp[i] = read64(&buffer); + } + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 4; j++) { + pkt->accel[i][j] = read32(&buffer); + } + } + + for (int i = 0; i < 4; i++) { + pkt->video_timestamp[i] = read64(&buffer); + } + + return; +} + +static bool +hololens_sensors_read_packets(struct wmr_hmd *wh) +{ + WMR_TRACE(wh, " "); + + unsigned char buffer[WMR_FEATURE_BUFFER_SIZE]; + + // Block for 100ms + int size = os_hid_read(wh->hid_hololens_sensors_dev, buffer, sizeof(buffer), 100); + + if (size < 0) { + WMR_ERROR(wh, "Error reading from device"); + return false; + } else if (size == 0) { + WMR_TRACE(wh, "No more data to read"); + return true; // No more messages, return. + } else { + WMR_TRACE(wh, "Read %u bytes", size); + } + + switch (buffer[0]) { + case WMR_MS_HOLOLENS_MSG_SENSORS: { + // Get the timing as close to reading the packet as possible. + uint64_t now_ns = os_monotonic_get_ns(); + + hololens_sensors_decode_packet(wh, &wh->packet, buffer, size); + + struct xrt_vec3 raw_gyro[4]; + struct xrt_vec3 raw_accel[4]; + + for (int i = 0; i < 4; i++) { + struct xrt_vec3 sample; + vec3_from_hololens_gyro(wh->packet.gyro, i, &sample); + math_quat_rotate_vec3(&wh->gyro_to_centerline.orientation, &sample, &raw_gyro[i]); + + vec3_from_hololens_accel(wh->packet.accel, i, &sample); + math_quat_rotate_vec3(&wh->accel_to_centerline.orientation, &sample, &raw_accel[i]); + } + + os_mutex_lock(&wh->fusion.mutex); + for (int i = 0; i < 4; i++) { + m_imu_3dof_update( // + &wh->fusion.i3dof, // + wh->packet.gyro_timestamp[i] * WMR_MS_HOLOLENS_NS_PER_TICK, // + &raw_accel[i], // + &raw_gyro[i]); // + } + wh->fusion.last_imu_timestamp_ns = now_ns; + wh->fusion.last_angular_velocity = raw_gyro[3]; + os_mutex_unlock(&wh->fusion.mutex); + + break; + } + case WMR_MS_HOLOLENS_MSG_UNKNOWN_05: + case WMR_MS_HOLOLENS_MSG_UNKNOWN_06: + case WMR_MS_HOLOLENS_MSG_UNKNOWN_0E: // + hololens_unknown_05_06_0E_decode_packet(wh, buffer, size); + break; + case WMR_MS_HOLOLENS_MSG_UNKNOWN_17: // + hololens_unknown_17_decode_packet(wh, buffer, size); + break; + case WMR_MS_HOLOLENS_MSG_CONTROL: + case WMR_MS_HOLOLENS_MSG_DEBUG: // + break; + default: // + WMR_DEBUG(wh, "Unknown message type: %02x, (%i)", buffer[0], size); + break; + } + + return true; +} + + +/* + * + * Control packets. + * + */ + +static void +control_ipd_value_decode(struct wmr_hmd *wh, const unsigned char *buffer, int size) +{ + if (size != 4) { + WMR_ERROR(wh, "Invalid control ipd distance packet size (expected 4 but got %i)", size); + return; + } + + uint8_t id = read8(&buffer); + uint8_t unknown = read8(&buffer); + uint16_t value = read16(&buffer); + + (void)id; + (void)unknown; + + wh->raw_ipd = value; + + WMR_DEBUG(wh, "Got IPD value: %04x", value); +} + +static bool +control_read_packets(struct wmr_hmd *wh) +{ + unsigned char buffer[WMR_FEATURE_BUFFER_SIZE]; + + // Do not block + int size = os_hid_read(wh->hid_control_dev, buffer, sizeof(buffer), 0); + + if (size < 0) { + WMR_ERROR(wh, "Error reading from device"); + return false; + } else if (size == 0) { + WMR_TRACE(wh, "No more data to read"); + return true; // No more messages, return. + } else { + WMR_TRACE(wh, "Read %u bytes", size); + } + + switch (buffer[0]) { + case WMR_CONTROL_MSG_IPD_VALUE: // + control_ipd_value_decode(wh, buffer, size); + break; + case WMR_CONTROL_MSG_UNKNOWN_05: // + break; + default: // + WMR_DEBUG(wh, "Unknown message type: %02x, (%i)", buffer[0], size); + break; + } + + return true; +} + + +/* + * + * Helpers and internal functions. + * + */ + +static void * +wmr_run_thread(void *ptr) +{ + struct wmr_hmd *wh = (struct wmr_hmd *)ptr; + + os_thread_helper_lock(&wh->oth); + while (os_thread_helper_is_running_locked(&wh->oth)) { + os_thread_helper_unlock(&wh->oth); + + // Does not block. + if (!control_read_packets(wh)) { + break; + } + + // Does block for a bit. + if (!hololens_sensors_read_packets(wh)) { + break; + } + } + + WMR_DEBUG(wh, "Exiting reading thread."); + + return NULL; +} + +static void +hololens_sensors_enable_imu(struct wmr_hmd *wh) +{ + int size = os_hid_write(wh->hid_hololens_sensors_dev, hololens_sensors_imu_on, sizeof(hololens_sensors_imu_on)); + if (size <= 0) { + WMR_ERROR(wh, "Error writing to device"); + return; + } +} + +#define HID_SEND(HID, DATA, STR) \ + do { \ + int _ret = os_hid_set_feature(HID, DATA, sizeof(DATA)); \ + if (_ret < 0) { \ + WMR_ERROR(wh, "Send (%s): %i", STR, _ret); \ + } \ + } while (false); + +#define HID_GET(HID, DATA, STR) \ + do { \ + int _ret = os_hid_get_feature(HID, DATA[0], DATA, sizeof(DATA)); \ + if (_ret < 0) { \ + WMR_ERROR(wh, "Get (%s): %i", STR, _ret); \ + } \ + } while (false); + +static int +wmr_hmd_activate_reverb(struct wmr_hmd *wh) +{ + struct os_hid_device *hid = wh->hid_control_dev; + + WMR_TRACE(wh, "Activating HP Reverb G1/G2 HMD..."); + + + // Hack to power up the Reverb G1 display, thanks to OpenHMD contibutors. + // Sleep before we start seems to improve reliability. + // 300ms is what Windows seems to do, so cargo cult that. + os_nanosleep(U_TIME_1MS_IN_NS * 300); + + for (int i = 0; i < 4; i++) { + unsigned char cmd[64] = {0x50, 0x01}; + HID_SEND(hid, cmd, "loop"); + + unsigned char data[64] = {0x50}; + HID_GET(hid, data, "loop"); + + os_nanosleep(U_TIME_1MS_IN_NS * 10); // Sleep 10ms + } + + unsigned char data[64] = {0x09}; + HID_GET(hid, data, "data_1"); + + data[0] = 0x08; + HID_GET(hid, data, "data_2"); + + data[0] = 0x06; + HID_GET(hid, data, "data_3"); + + // Wake up the display. + unsigned char cmd[2] = {0x04, 0x01}; + HID_SEND(hid, cmd, "screen_on"); + + WMR_INFO(wh, "Sent activation report, sleeping for compositor."); + + /* + * Sleep so display completes power up and modes be enumerated. + * Two seconds seems to be needed, 1 was not enough. + */ + os_nanosleep(U_TIME_1MS_IN_NS * 2000); + + + return 0; +} + +static void +wmr_hmd_deactivate_reverb(struct wmr_hmd *wh) +{ + struct os_hid_device *hid = wh->hid_control_dev; + + /* Turn the screen off */ + unsigned char cmd[2] = {0x04, 0x00}; + HID_SEND(hid, cmd, "screen_off"); +} + + +/* + * + * Config functions. + * + */ + +static int +wmr_config_command_sync(struct wmr_hmd *wh, unsigned char type, unsigned char *buf, int len) +{ + struct os_hid_device *hid = wh->hid_hololens_sensors_dev; + + unsigned char cmd[64] = {0x02, type}; + os_hid_write(hid, cmd, sizeof(cmd)); + + do { + int size = os_hid_read(hid, buf, len, -1); + if (size == -1) { + return -1; + } + if (buf[0] == WMR_MS_HOLOLENS_MSG_CONTROL) { + return size; + } + } while (buf[0] == WMR_MS_HOLOLENS_MSG_SENSORS || // + buf[0] == WMR_MS_HOLOLENS_MSG_DEBUG || // + buf[0] == WMR_MS_HOLOLENS_MSG_UNKNOWN_17); + + return -1; +} + +static int +wmr_read_config_part(struct wmr_hmd *wh, unsigned char type, unsigned char *data, int len) +{ + + unsigned char buf[33]; + int offset = 0; + int size; + + size = wmr_config_command_sync(wh, 0x0b, buf, sizeof(buf)); + if (size != 33 || buf[0] != 0x02) { + WMR_ERROR(wh, "Failed to issue command 0b: %02x %02x %02x", buf[0], buf[1], buf[2]); + return -1; + } + + size = wmr_config_command_sync(wh, type, buf, sizeof(buf)); + if (size != 33 || buf[0] != 0x02) { + WMR_ERROR(wh, "Failed to issue command %02x: %02x %02x %02x", type, buf[0], buf[1], buf[2]); + return -1; + } + + while (true) { + size = wmr_config_command_sync(wh, 0x08, buf, sizeof(buf)); + if (size != 33 || (buf[1] != 0x01 && buf[1] != 0x02)) { + WMR_ERROR(wh, "Failed to issue command 08: %02x %02x %02x", buf[0], buf[1], buf[2]); + return -1; + } + + if (buf[1] != 0x01) { + break; + } + + if (buf[2] > len || offset + buf[2] > len) { + WMR_ERROR(wh, "Getting more information then requested"); + return -1; + } + + memcpy(data + offset, buf + 3, buf[2]); + offset += buf[2]; + } + + return offset; +} + +XRT_MAYBE_UNUSED static int +wmr_read_config_raw(struct wmr_hmd *wh, uint8_t **out_data, size_t *out_size) +{ + unsigned char meta[84]; + uint8_t *data; + int size, data_size; + + size = wmr_read_config_part(wh, 0x06, meta, sizeof(meta)); + WMR_DEBUG(wh, "(0x06, meta) => %d", size); + + if (size < 0) { + return -1; + } + + /* + * No idea what the other 64 bytes of metadata are, but the first two + * seem to be little endian size of the data store. + */ + data_size = meta[0] | (meta[1] << 8); + data = calloc(1, data_size + 1); + if (!data) { + return -1; + } + data[data_size] = '\0'; + + size = wmr_read_config_part(wh, 0x04, data, data_size); + WMR_DEBUG(wh, "(0x04, data) => %d", size); + if (size < 0) { + free(data); + return -1; + } + + WMR_DEBUG(wh, "Read %d-byte config data", data_size); + + *out_data = data; + *out_size = size; + + return 0; +} + +static int +wmr_read_config(struct wmr_hmd *wh) +{ + unsigned char *data = NULL, *config_json_block; + size_t data_size; + int ret; + + // Read config + ret = wmr_read_config_raw(wh, &data, &data_size); + if (ret < 0) + return ret; + + /* De-obfuscate the JSON config */ + /* FIXME: The header contains little-endian values that need swapping for big-endian */ + struct wmr_config_header *hdr = (struct wmr_config_header *)data; + + /* Take a copy of the header */ + memcpy(&wh->config_hdr, hdr, sizeof(struct wmr_config_header)); + + WMR_INFO(wh, "Manufacturer: %.*s", (int)sizeof(hdr->manufacturer), hdr->manufacturer); + WMR_INFO(wh, "Device: %.*s", (int)sizeof(hdr->device), hdr->device); + WMR_INFO(wh, "Serial: %.*s", (int)sizeof(hdr->serial), hdr->serial); + WMR_INFO(wh, "UID: %.*s", (int)sizeof(hdr->uid), hdr->uid); + WMR_INFO(wh, "Name: %.*s", (int)sizeof(hdr->name), hdr->name); + WMR_INFO(wh, "Revision: %.*s", (int)sizeof(hdr->revision), hdr->revision); + WMR_INFO(wh, "Revision Date: %.*s", (int)sizeof(hdr->revision_date), hdr->revision_date); + + snprintf(wh->base.str, XRT_DEVICE_NAME_LEN, "%.*s", (int)sizeof(hdr->name), hdr->name); + + if (hdr->json_start >= data_size || (data_size - hdr->json_start) < hdr->json_size) { + WMR_ERROR(wh, "Invalid WMR config block - incorrect sizes"); + free(data); + return -1; + } + + config_json_block = data + hdr->json_start + sizeof(uint16_t); + for (unsigned int i = 0; i < hdr->json_size - sizeof(uint16_t); i++) { + config_json_block[i] ^= wmr_config_key[i % sizeof(wmr_config_key)]; + } + + if (!wmr_config_parse(&wh->config, (char *)config_json_block, wh->log_level)) { + free(data); + return -1; + } + + free(data); + return 0; +} + +/* + * + * Device members. + * + */ + +static void +wmr_hmd_update_inputs(struct xrt_device *xdev) +{ + struct wmr_hmd *wh = wmr_hmd(xdev); + (void)wh; +} + +static void +wmr_hmd_get_tracked_pose(struct xrt_device *xdev, + enum xrt_input_name name, + uint64_t at_timestamp_ns, + struct xrt_space_relation *out_relation) +{ + struct wmr_hmd *wh = wmr_hmd(xdev); + + if (name != XRT_INPUT_GENERIC_HEAD_POSE) { + WMR_ERROR(wh, "Unknown input name"); + return; + } + + // Variables needed for prediction. + uint64_t last_imu_timestamp_ns = 0; + struct xrt_space_relation relation = {0}; + relation.relation_flags = (enum xrt_space_relation_flags)( // + XRT_SPACE_RELATION_ANGULAR_VELOCITY_VALID_BIT | // + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | // + XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT); + + // Get data while holding the lock. + os_mutex_lock(&wh->fusion.mutex); + relation.pose.orientation = wh->fusion.i3dof.rot; + relation.angular_velocity = wh->fusion.last_angular_velocity; + last_imu_timestamp_ns = wh->fusion.last_imu_timestamp_ns; + os_mutex_unlock(&wh->fusion.mutex); + + // No prediction needed. + if (at_timestamp_ns < last_imu_timestamp_ns) { + *out_relation = relation; + return; + } + + uint64_t prediction_ns = at_timestamp_ns - last_imu_timestamp_ns; + double prediction_s = time_ns_to_s(prediction_ns); + + m_predict_relation(&relation, prediction_s, out_relation); +} + +static void +wmr_hmd_get_view_pose(struct xrt_device *xdev, + const struct xrt_vec3 *eye_relation, + uint32_t view_index, + struct xrt_pose *out_pose) +{ + (void)xdev; + u_device_get_view_pose(eye_relation, view_index, out_pose); +} + +static void +wmr_hmd_destroy(struct xrt_device *xdev) +{ + struct wmr_hmd *wh = wmr_hmd(xdev); + + // Destroy the thread object. + os_thread_helper_destroy(&wh->oth); + + if (wh->hid_hololens_sensors_dev != NULL) { + os_hid_destroy(wh->hid_hololens_sensors_dev); + wh->hid_hololens_sensors_dev = NULL; + } + + if (wh->hid_control_dev != NULL) { + /* Do any deinit if we have a deinit function */ + if (wh->hmd_desc && wh->hmd_desc->deinit_func) { + wh->hmd_desc->deinit_func(wh); + } + os_hid_destroy(wh->hid_control_dev); + wh->hid_control_dev = NULL; + } + + // Destroy the fusion. + m_imu_3dof_close(&wh->fusion.i3dof); + + os_mutex_destroy(&wh->fusion.mutex); + + free(wh); +} + +static bool +compute_distortion_wmr(struct xrt_device *xdev, int view, float u, float v, struct xrt_uv_triplet *result) +{ + struct wmr_hmd *wh = wmr_hmd(xdev); + + assert(view == 0 || view == 1); + + const struct wmr_distortion_eye_config *ec = wh->config.eye_params + view; + struct wmr_hmd_distortion_params *distortion_params = wh->distortion_params + view; + + // Results r/g/b. + struct xrt_vec2 tc[3]; + + // Dear compiler, please vectorize. + for (int i = 0; i < 3; i++) { + const struct wmr_distortion_3K *distortion3K = ec->distortion3K + i; + + /* Scale the 0..1 input UV back to pixels relative to the distortion center, + * accounting for the right eye starting at X = panel_width / 2.0 */ + struct xrt_vec2 pix_coord = {(u + 1.0 * view) * (ec->display_size.x / 2.0) - distortion3K->eye_center.x, + v * ec->display_size.y - distortion3K->eye_center.y}; + + float r2 = m_vec2_dot(pix_coord, pix_coord); + float k1 = distortion3K->k[0]; + float k2 = distortion3K->k[1]; + float k3 = distortion3K->k[2]; + + float d = 1.0 + r2 * (k1 + r2 * (k2 + r2 * k3)); + + /* Map the distorted pixel coordinate back to normalised view plane coords using the inverse affine + * xform */ + struct xrt_vec3 p = {(pix_coord.x * d + distortion3K->eye_center.x), + (pix_coord.y * d + distortion3K->eye_center.y), 1.0}; + struct xrt_vec3 vp; + math_matrix_3x3_transform_vec3(&distortion_params->inv_affine_xform, &p, &vp); + + /* Finally map back to the input texture 0..1 range based on the render FoV (from tex_N_range.x .. + * tex_N_range.y) */ + tc[i].x = ((vp.x / vp.z) - distortion_params->tex_x_range.x) / + (distortion_params->tex_x_range.y - distortion_params->tex_x_range.x); + tc[i].y = ((vp.y / vp.z) - distortion_params->tex_y_range.x) / + (distortion_params->tex_y_range.y - distortion_params->tex_y_range.x); + } + + result->r = tc[0]; + result->g = tc[1]; + result->b = tc[2]; + + return true; +} + +/* + * Compute the visible area bounds by calculating the X/Y limits of a + * crosshair through the distortion center, and back-project to the render FoV, + */ +static void +compute_distortion_bounds(struct wmr_hmd *wh, + int view, + float *out_angle_left, + float *out_angle_right, + float *out_angle_down, + float *out_angle_up) +{ + assert(view == 0 || view == 1); + + float tanangle_left = 0.0, tanangle_right = 0.0, tanangle_up = 0.0, tanangle_down = 0.0; + + const struct wmr_distortion_eye_config *ec = wh->config.eye_params + view; + struct wmr_hmd_distortion_params *distortion_params = wh->distortion_params + view; + + for (int i = 0; i < 3; i++) { + const struct wmr_distortion_3K *distortion3K = ec->distortion3K + i; + + /* The X coords start at 0 for the left eye, and display_size.x / 2.0 for the right */ + const struct xrt_vec2 pix_coords[4] = { + /* -eye_center_x, 0 */ + {(1.0 * view) * (ec->display_size.x / 2.0) - distortion3K->eye_center.x, 0.0}, + /* 0, -eye_center_y */ + {0.0, -distortion3K->eye_center.y}, + /* width-eye_center_x, 0 */ + {(1.0 + 1.0 * view) * (ec->display_size.x / 2.0) - distortion3K->eye_center.x, 0.0}, + /* 0, height-eye_center_y */ + {0.0, ec->display_size.y - distortion3K->eye_center.y}, + }; + + for (int c = 0; c < 4; c++) { + const struct xrt_vec2 pix_coord = pix_coords[c]; + + float k1 = distortion3K->k[0]; + float k2 = distortion3K->k[1]; + float k3 = distortion3K->k[2]; + + float r2 = m_vec2_dot(pix_coord, pix_coord); + + /* distort the pixel */ + float d = 1.0 + r2 * (k1 + r2 * (k2 + r2 * k3)); + + /* Map the distorted pixel coordinate back to normalised view plane coords using the inverse + * affine xform */ + struct xrt_vec3 p = {(pix_coord.x * d + distortion3K->eye_center.x), + (pix_coord.y * d + distortion3K->eye_center.y), 1.0}; + struct xrt_vec3 vp; + + math_matrix_3x3_transform_vec3(&distortion_params->inv_affine_xform, &p, &vp); + vp.x /= vp.z; + vp.y /= vp.z; + + if (pix_coord.x < 0.0) { + if (vp.x < tanangle_left) + tanangle_left = vp.x; + } else { + if (vp.x > tanangle_right) + tanangle_right = vp.x; + } + + if (pix_coord.y < 0.0) { + if (vp.y < tanangle_up) + tanangle_up = vp.y; + } else { + if (vp.y > tanangle_down) + tanangle_down = vp.y; + } + + WMR_DEBUG(wh, "channel %d delta coord %f, %f d pixel %f %f, %f -> %f, %f", i, pix_coord.x, + pix_coord.y, d, p.x, p.y, vp.x, vp.y); + } + } + + *out_angle_left = atan(tanangle_left); + *out_angle_right = atan(tanangle_right); + *out_angle_down = -atan(tanangle_down); + *out_angle_up = -atan(tanangle_up); +} + +struct xrt_device * +wmr_hmd_create(enum wmr_headset_type hmd_type, + struct os_hid_device *hid_holo, + struct os_hid_device *hid_ctrl, + enum u_logging_level ll) +{ + enum u_device_alloc_flags flags = + (enum u_device_alloc_flags)(U_DEVICE_ALLOC_HMD | U_DEVICE_ALLOC_TRACKING_NONE); + int ret = 0, i; + int eye; + + struct wmr_hmd *wh = U_DEVICE_ALLOCATE(struct wmr_hmd, flags, 1, 0); + if (!wh) { + return NULL; + } + + // Populate the base members. + wh->base.update_inputs = wmr_hmd_update_inputs; + wh->base.get_tracked_pose = wmr_hmd_get_tracked_pose; + wh->base.get_view_pose = wmr_hmd_get_view_pose; + wh->base.destroy = wmr_hmd_destroy; + wh->base.name = XRT_DEVICE_GENERIC_HMD; + wh->base.device_type = XRT_DEVICE_TYPE_HMD; + wh->log_level = ll; + + wh->base.orientation_tracking_supported = true; + wh->base.position_tracking_supported = false; + wh->base.hand_tracking_supported = false; + wh->hid_hololens_sensors_dev = hid_holo; + wh->hid_control_dev = hid_ctrl; + + // Mutex before thread. + ret = os_mutex_init(&wh->fusion.mutex); + if (ret != 0) { + WMR_ERROR(wh, "Failed to init mutex!"); + wmr_hmd_destroy(&wh->base); + wh = NULL; + return NULL; + } + + // Thread and other state. + ret = os_thread_helper_init(&wh->oth); + if (ret != 0) { + WMR_ERROR(wh, "Failed to init threading!"); + wmr_hmd_destroy(&wh->base); + wh = NULL; + return NULL; + } + + // Setup input. + wh->base.inputs[0].name = XRT_INPUT_GENERIC_HEAD_POSE; + + // Read config file from HMD + if (wmr_read_config(wh) < 0) { + WMR_ERROR(wh, "Failed to load headset configuration!"); + wmr_hmd_destroy(&wh->base); + wh = NULL; + return NULL; + } + + /* Now that we have the config loaded, iterate the map of known headsets and see if we have + * an entry for this specific headset (otherwise the generic entry will be used) + */ + for (i = 0; i < headset_map_n; i++) { + const struct wmr_headset_descriptor *cur = &headset_map[i]; + + if (hmd_type == cur->hmd_type) { + wh->hmd_desc = cur; + if (hmd_type != WMR_HEADSET_GENERIC) + break; /* Stop checking if we have a specific match, or keep going for the GENERIC + catch-all type */ + } + + if (cur->dev_id_str && strncmp(wh->config_hdr.name, cur->dev_id_str, 64) == 0) { + hmd_type = cur->hmd_type; + wh->hmd_desc = cur; + break; + } + } + assert(wh->hmd_desc != NULL); /* We must have matched something, or the map is set up wrong */ + + WMR_INFO(wh, "Found WMR headset type: %s", wh->hmd_desc->debug_name); + + m_imu_3dof_init(&wh->fusion.i3dof, M_IMU_3DOF_USE_GRAVITY_DUR_20MS); + + // Setup variable tracker. + u_var_add_root(wh, "WMR HMD", true); + u_var_add_gui_header(wh, &wh->gui.fusion, "3DoF Fusion"); + m_imu_3dof_add_vars(&wh->fusion.i3dof, wh, ""); + u_var_add_gui_header(wh, &wh->gui.misc, "Misc"); + u_var_add_log_level(wh, &wh->log_level, "log_level"); + + // Compute centerline in the HMD's calibration coordinate space as the average of the two display poses + math_quat_slerp(&wh->config.eye_params[0].pose.orientation, &wh->config.eye_params[1].pose.orientation, 0.5f, + &wh->centerline.orientation); + wh->centerline.position.x = + (wh->config.eye_params[0].pose.position.x + wh->config.eye_params[1].pose.position.x) * 0.5f; + wh->centerline.position.y = + (wh->config.eye_params[0].pose.position.y + wh->config.eye_params[1].pose.position.y) * 0.5f; + wh->centerline.position.z = + (wh->config.eye_params[0].pose.position.z + wh->config.eye_params[1].pose.position.z) * 0.5f; + + // Compute display and sensor offsets relative to the centerline + for (int dIdx = 0; dIdx < 2; ++dIdx) { + math_pose_invert(&wh->config.eye_params[dIdx].pose, &wh->display_to_centerline[dIdx]); + math_pose_transform(&wh->centerline, &wh->display_to_centerline[dIdx], + &wh->display_to_centerline[dIdx]); + } + math_pose_invert(&wh->config.accel_pose, &wh->accel_to_centerline); + math_pose_transform(&wh->centerline, &wh->accel_to_centerline, &wh->accel_to_centerline); + math_pose_invert(&wh->config.gyro_pose, &wh->gyro_to_centerline); + math_pose_transform(&wh->centerline, &wh->gyro_to_centerline, &wh->gyro_to_centerline); + math_pose_invert(&wh->config.mag_pose, &wh->mag_to_centerline); + math_pose_transform(&wh->centerline, &wh->mag_to_centerline, &wh->mag_to_centerline); + + struct u_device_simple_info info; + info.display.w_pixels = wh->config.eye_params[0].display_size.x; + info.display.h_pixels = wh->config.eye_params[0].display_size.y; + + info.lens_horizontal_separation_meters = + fabs(wh->display_to_centerline[1].position.x - wh->display_to_centerline[0].position.x); + + /* We set up a dummy side-by-side config, then adjust the actual FoV bounds + * in compute_distortion_bounds() below */ + info.display.w_meters = 0.13f; + info.display.h_meters = 0.07f; + info.lens_vertical_position_meters = 0.07f / 2.0f; + info.views[0].fov = 85.0f * (M_PI / 180.0f); + info.views[1].fov = 85.0f * (M_PI / 180.0f); + + if (!u_device_setup_split_side_by_side(&wh->base, &info)) { + WMR_ERROR(wh, "Failed to setup basic HMD device info"); + wmr_hmd_destroy(&wh->base); + wh = NULL; + return NULL; + } + + // Distortion information, fills in xdev->compute_distortion(). + for (eye = 0; eye < 2; eye++) { + math_matrix_3x3_inverse(&wh->config.eye_params[eye].affine_xform, + &wh->distortion_params[eye].inv_affine_xform); + + compute_distortion_bounds( + wh, eye, &wh->base.hmd->views[eye].fov.angle_left, &wh->base.hmd->views[eye].fov.angle_right, + &wh->base.hmd->views[eye].fov.angle_down, &wh->base.hmd->views[eye].fov.angle_up); + + WMR_INFO(wh, "FoV eye %d angles left %f right %f down %f up %f", eye, + wh->base.hmd->views[eye].fov.angle_left, wh->base.hmd->views[eye].fov.angle_right, + wh->base.hmd->views[eye].fov.angle_down, wh->base.hmd->views[eye].fov.angle_up); + + wh->distortion_params[eye].tex_x_range.x = tan(wh->base.hmd->views[eye].fov.angle_left); + wh->distortion_params[eye].tex_x_range.y = tan(wh->base.hmd->views[eye].fov.angle_right); + wh->distortion_params[eye].tex_y_range.x = tan(wh->base.hmd->views[eye].fov.angle_down); + wh->distortion_params[eye].tex_y_range.y = tan(wh->base.hmd->views[eye].fov.angle_up); + + WMR_INFO(wh, "Render texture range %f, %f to %f, %f", wh->distortion_params[eye].tex_x_range.x, + wh->distortion_params[eye].tex_y_range.x, wh->distortion_params[eye].tex_x_range.y, + wh->distortion_params[eye].tex_y_range.y); + } + + wh->base.hmd->distortion.models = XRT_DISTORTION_MODEL_COMPUTE; + wh->base.hmd->distortion.preferred = XRT_DISTORTION_MODEL_COMPUTE; + wh->base.compute_distortion = compute_distortion_wmr; + u_distortion_mesh_fill_in_compute(&wh->base); + + /* We're set up. Activate the HMD and turn on the IMU */ + if (wh->hmd_desc->init_func && wh->hmd_desc->init_func(wh) != 0) { + WMR_ERROR(wh, "Activation of HMD failed"); + wmr_hmd_destroy(&wh->base); + wh = NULL; + return NULL; + } + + // Switch on IMU on the HMD. + hololens_sensors_enable_imu(wh); + + + // Hand over hololens sensor device to reading thread. + ret = os_thread_helper_start(&wh->oth, wmr_run_thread, wh); + if (ret != 0) { + WMR_ERROR(wh, "Failed to start thread!"); + wmr_hmd_destroy(&wh->base); + wh = NULL; + return NULL; + } + + return &wh->base; +} diff --git a/src/xrt/drivers/wmr/wmr_hmd.h b/src/xrt/drivers/wmr/wmr_hmd.h new file mode 100644 index 000000000..b9a210957 --- /dev/null +++ b/src/xrt/drivers/wmr/wmr_hmd.h @@ -0,0 +1,170 @@ +// Copyright 2018, Philipp Zabel. +// Copyright 2020-2021, N Madsen. +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Interface to the WMR HMD driver code. + * @author Philipp Zabel + * @author nima01 + * @author Jakob Bornecrantz + * @ingroup drv_wmr + */ + +#pragma once + +#include "xrt/xrt_device.h" +#include "xrt/xrt_prober.h" +#include "os/os_threading.h" +#include "math/m_imu_3dof.h" +#include "util/u_logging.h" +#include "util/u_distortion_mesh.h" + +#include "wmr_protocol.h" +#include "wmr_config.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +enum rvb_g1_status_bits +{ + // clang-format off + REVERB_G1_STATUS_BIT_UNKNOWN_BIT_0 = (1 << 0), + REVERB_G1_STATUS_BIT_UNKNOWN_BIT_1 = (1 << 1), + REVERB_G1_STATUS_BIT_UNKNOWN_BIT_2 = (1 << 2), + REVERB_G1_STATUS_BIT_UNKNOWN_BIT_3 = (1 << 3), + REVERB_G1_STATUS_BIT_UNKNOWN_BIT_4 = (1 << 4), + REVERB_G1_STATUS_BIT_UNKNOWN_BIT_5 = (1 << 5), + REVERB_G1_STATUS_BIT_UNKNOWN_BIT_6 = (1 << 6), + REVERB_G1_STATUS_BIT_UNKNOWN_BIT_7 = (1 << 7), + // clang-format on +}; + +enum wmr_headset_type +{ + WMR_HEADSET_GENERIC, + WMR_HEADSET_REVERB_G1, + WMR_HEADSET_REVERB_G2, + WMR_HEADSET_SAMSUNG_800ZAA, + WMR_HEADSET_LENOVO_EXPLORER +}; + +struct wmr_hmd; + +struct wmr_headset_descriptor +{ + enum wmr_headset_type hmd_type; + + /* String by which we recognise the device */ + const char *dev_id_str; + /* Friendly ID string for debug */ + const char *debug_name; + + int (*init_func)(struct wmr_hmd *wh); + void (*deinit_func)(struct wmr_hmd *wh); +}; + +struct wmr_hmd_distortion_params +{ + /* Inverse affine transform to move from (undistorted) pixels + * to image plane / normalised image coordinates + */ + struct xrt_matrix_3x3 inv_affine_xform; + + /* tan(angle) FoV min/max for X and Y in the input texture */ + struct xrt_vec2 tex_x_range; + struct xrt_vec2 tex_y_range; +}; + +/*! + * @implements xrt_device + */ +struct wmr_hmd +{ + struct xrt_device base; + + const struct wmr_headset_descriptor *hmd_desc; + + /* firmware configuration block, with device names etc */ + struct wmr_config_header config_hdr; + + /* Config data parsed from the firmware JSON */ + struct wmr_hmd_config config; + + //! Packet reading thread. + struct os_thread_helper oth; + + enum u_logging_level log_level; + + /*! + * This is the hololens sensor device, this is were we get all of the + * IMU data and read the config from. + * + * During start it is owned by the thread creating the device, after + * init it is owned by the reading thread, there is no mutex protecting + * this field as it's only used by the reading thread in @p oth. + */ + struct os_hid_device *hid_hololens_sensors_dev; + struct os_hid_device *hid_control_dev; + + //! Latest raw IPD value from the device. + uint16_t raw_ipd; + + /* Distortion related parameters */ + struct wmr_hmd_distortion_params distortion_params[2]; + + // Config-derived poses + struct xrt_pose centerline; + struct xrt_pose display_to_centerline[2]; + struct xrt_pose accel_to_centerline; + struct xrt_pose gyro_to_centerline; + struct xrt_pose mag_to_centerline; + + struct hololens_sensors_packet packet; + + struct + { + //! Protects all members of the `fusion` substruct. + struct os_mutex mutex; + + //! Main fusion calculator. + struct m_imu_3dof i3dof; + + //! The last angular velocity from the IMU, for prediction. + struct xrt_vec3 last_angular_velocity; + + //! When did we get the last IMU sample, in CPU time. + uint64_t last_imu_timestamp_ns; + } fusion; + + struct + { + bool fusion; + bool misc; + } gui; +}; + +static inline struct wmr_hmd * +wmr_hmd(struct xrt_device *p) +{ + return (struct wmr_hmd *)p; +} + +struct xrt_device * +wmr_hmd_create(enum wmr_headset_type hmd_type, + struct os_hid_device *hid_holo, + struct os_hid_device *hid_ctrl, + enum u_logging_level ll); + +#define WMR_TRACE(d, ...) U_LOG_XDEV_IFL_T(&d->base, d->log_level, __VA_ARGS__) +#define WMR_DEBUG(d, ...) U_LOG_XDEV_IFL_D(&d->base, d->log_level, __VA_ARGS__) +#define WMR_INFO(d, ...) U_LOG_XDEV_IFL_I(&d->base, d->log_level, __VA_ARGS__) +#define WMR_WARN(d, ...) U_LOG_XDEV_IFL_W(&d->base, d->log_level, __VA_ARGS__) +#define WMR_ERROR(d, ...) U_LOG_XDEV_IFL_E(&d->base, d->log_level, __VA_ARGS__) + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/wmr/wmr_interface.h b/src/xrt/drivers/wmr/wmr_interface.h new file mode 100644 index 000000000..c6821e1f5 --- /dev/null +++ b/src/xrt/drivers/wmr/wmr_interface.h @@ -0,0 +1,48 @@ +// Copyright 2020-2021, N Madsen. +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Interface to the WMR driver. + * @author nima01 + * @author Jakob Bornecrantz + * @ingroup drv_wmr + */ + +#pragma once + + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * @defgroup drv_wmr Windows Mixed Reality driver + * @ingroup drv + * + * @brief Windows Mixed Reality driver. + */ + +/*! + * Probing function for Windows Mixed Reality devices. + * + * @ingroup drv_wmr + */ +int +wmr_found(struct xrt_prober *xp, + struct xrt_prober_device **devices, + size_t num_devices, + size_t index, + cJSON *attached_data, + struct xrt_device **out_xdev); + +/*! + * @dir drivers/wmr + * + * @brief @ref drv_wmr files. + */ + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/wmr/wmr_prober.c b/src/xrt/drivers/wmr/wmr_prober.c new file mode 100644 index 000000000..48ddf165c --- /dev/null +++ b/src/xrt/drivers/wmr/wmr_prober.c @@ -0,0 +1,181 @@ +// Copyright 2020-2021, N Madsen. +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief WMR prober code. + * @author nima01 + * @author Jakob Bornecrantz + * @ingroup drv_wmr + */ + +#include "xrt/xrt_prober.h" + +#include "util/u_misc.h" +#include "util/u_debug.h" +#include "util/u_logging.h" + +#include "wmr_interface.h" +#include "wmr_hmd.h" +#include "wmr_common.h" + +#include +#include +#include + +/* + * + * Defines & structs. + * + */ + +DEBUG_GET_ONCE_LOG_OPTION(wmr_log, "WMR_LOG", U_LOGGING_INFO) + + +/* + * + * Functions. + * + */ + +static bool +check_and_get_interface_hp(struct xrt_prober_device *device, enum wmr_headset_type *out_hmd_type, int *out_interface) +{ + if (device->product_id != REVERB_G1_PID && device->product_id != REVERB_G2_PID) { + return false; + } + + if (device->product_id == REVERB_G1_PID) + *out_hmd_type = WMR_HEADSET_REVERB_G1; + else + *out_hmd_type = WMR_HEADSET_REVERB_G2; + *out_interface = 0; + + return true; +} + +static bool +check_and_get_interface_lenovo(struct xrt_prober_device *device, + enum wmr_headset_type *out_hmd_type, + int *out_interface) +{ + if (device->product_id != EXPLORER_PID) { + return false; + } + + *out_hmd_type = WMR_HEADSET_LENOVO_EXPLORER; + *out_interface = 0; + + return true; +} + +static bool +find_control_device(struct xrt_prober *xp, + struct xrt_prober_device **devices, + size_t num_devices, + enum u_logging_level ll, + enum wmr_headset_type *out_hmd_type, + struct xrt_prober_device **out_device, + int *out_interface) +{ + struct xrt_prober_device *dev = NULL; + int interface = 0; + + for (size_t i = 0; i < num_devices; i++) { + bool match = false; + + if (devices[i]->bus != XRT_BUS_TYPE_USB) { + continue; + } + + switch (devices[i]->vendor_id) { + case HP_VID: match = check_and_get_interface_hp(devices[i], out_hmd_type, &interface); break; + case LENOVO_VID: match = check_and_get_interface_lenovo(devices[i], out_hmd_type, &interface); break; + default: break; + } + + if (!match) { + continue; + } + + if (dev != NULL) { + U_LOG_IFL_W(ll, "Found multiple control devices, using the last."); + } + dev = devices[i]; + } + + unsigned char m_str[256] = {0}; + unsigned char p_str[256] = {0}; + xrt_prober_get_string_descriptor(xp, dev, XRT_PROBER_STRING_MANUFACTURER, m_str, sizeof(m_str)); + xrt_prober_get_string_descriptor(xp, dev, XRT_PROBER_STRING_PRODUCT, p_str, sizeof(p_str)); + + U_LOG_IFL_D(ll, "Found control device '%s' '%s' (%04X:%04X)", p_str, m_str, dev->vendor_id, dev->product_id); + + *out_device = dev; + *out_interface = interface; + + return dev != NULL; +} + + +/* + * + * Exported functions. + * + */ + +int +wmr_found(struct xrt_prober *xp, + struct xrt_prober_device **devices, + size_t num_devices, + size_t index, + cJSON *attached_data, + struct xrt_device **out_xdev) +{ + enum u_logging_level ll = debug_get_log_option_wmr_log(); + + struct xrt_prober_device *dev_holo = devices[index]; + struct xrt_prober_device *dev_ctrl = NULL; + enum wmr_headset_type hmd_type = WMR_HEADSET_GENERIC; + int interface_holo = 2; + int interface_ctrl = 0; + + unsigned char buf[256] = {0}; + int result = xrt_prober_get_string_descriptor(xp, dev_holo, XRT_PROBER_STRING_PRODUCT, buf, sizeof(buf)); + + if (!xrt_prober_match_string(xp, dev_holo, XRT_PROBER_STRING_MANUFACTURER, MS_HOLOLENS_MANUFACTURER_STRING) || + !xrt_prober_match_string(xp, dev_holo, XRT_PROBER_STRING_PRODUCT, MS_HOLOLENS_PRODUCT_STRING)) { + U_LOG_IFL_E(ll, "HoloLens Sensors manufacturer or product strings did not match."); + return -1; + } + + if (!find_control_device(xp, devices, num_devices, ll, &hmd_type, &dev_ctrl, &interface_ctrl)) { + U_LOG_IFL_E(ll, + "Did not find companion control device." + "\n\tCurrently only Reverb G1 and G2 is supported"); + return -1; + } + + struct os_hid_device *hid_holo = NULL; + result = xrt_prober_open_hid_interface(xp, dev_holo, interface_holo, &hid_holo); + if (result != 0) { + U_LOG_IFL_E(ll, "Failed to open HoloLens Sensors HID interface"); + return -1; + } + + struct os_hid_device *hid_ctrl = NULL; + result = xrt_prober_open_hid_interface(xp, dev_ctrl, interface_ctrl, &hid_ctrl); + if (result != 0) { + U_LOG_IFL_E(ll, "Failed to open HoloLens Control HID interface"); + return -1; + } + + struct xrt_device *p = wmr_hmd_create(hmd_type, hid_holo, hid_ctrl, ll); + if (!p) { + U_LOG_IFL_E(ll, "Failed to create Windows Mixed Reality device"); + return -1; + } + + *out_xdev = p; + return 1; +} diff --git a/src/xrt/drivers/wmr/wmr_protocol.c b/src/xrt/drivers/wmr/wmr_protocol.c new file mode 100644 index 000000000..da851294b --- /dev/null +++ b/src/xrt/drivers/wmr/wmr_protocol.c @@ -0,0 +1,60 @@ +// Copyright 2018, Philipp Zabel. +// Copyright 2020-2021, N Madsen. +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief WMR and MS HoloLens protocol helpers implementation. + * @author Philipp Zabel + * @author nima01 + * @ingroup drv_wmr + */ + +#include "wmr_protocol.h" + + +/* + * + * WMR and MS HoloLens Sensors protocol helpers + * + */ + +void +vec3_from_hololens_accel(int32_t sample[3][4], int i, struct xrt_vec3 *out_vec) +{ + out_vec->x = (float)sample[0][i] * 0.001f * 1.0f; + out_vec->y = (float)sample[1][i] * 0.001f * -1.0f; + out_vec->z = (float)sample[2][i] * 0.001f * -1.0f; +} + +void +vec3_from_hololens_gyro(int16_t sample[3][32], int i, struct xrt_vec3 *out_vec) +{ + out_vec->x = (float)(sample[0][8 * i + 0] + // + sample[0][8 * i + 1] + // + sample[0][8 * i + 2] + // + sample[0][8 * i + 3] + // + sample[0][8 * i + 4] + // + sample[0][8 * i + 5] + // + sample[0][8 * i + 6] + // + sample[0][8 * i + 7]) * + 0.001f * 0.125f; + out_vec->y = (float)(sample[1][8 * i + 0] + // + sample[1][8 * i + 1] + // + sample[1][8 * i + 2] + // + sample[1][8 * i + 3] + // + sample[1][8 * i + 4] + // + sample[1][8 * i + 5] + // + sample[1][8 * i + 6] + // + sample[1][8 * i + 7]) * + 0.001f * -0.125f; + out_vec->z = (float)(sample[2][8 * i + 0] + // + sample[2][8 * i + 1] + // + sample[2][8 * i + 2] + // + sample[2][8 * i + 3] + // + sample[2][8 * i + 4] + // + sample[2][8 * i + 5] + // + sample[2][8 * i + 6] + // + sample[2][8 * i + 7]) * + 0.001f * -0.125f; +} diff --git a/src/xrt/drivers/wmr/wmr_protocol.h b/src/xrt/drivers/wmr/wmr_protocol.h new file mode 100644 index 000000000..b9635272a --- /dev/null +++ b/src/xrt/drivers/wmr/wmr_protocol.h @@ -0,0 +1,142 @@ +// Copyright 2018, Philipp Zabel. +// Copyright 2020-2021, N Madsen. +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief WMR and MS HoloLens protocol constants, structures and helpers header + * @author Philipp Zabel + * @author nima01 + * @ingroup drv_wmr + */ + +#pragma once + +#include "math/m_vec2.h" + + +#ifdef __cplusplus +extern "C" { +#endif + + +/*! + * WMR and MS HoloLens Sensors protocol constants and structures + * + * @ingroup drv_wmr + * @{ + */ + +#define WMR_FEATURE_BUFFER_SIZE 497 +#define WMR_MS_HOLOLENS_NS_PER_TICK 100 + +#define WMR_MS_HOLOLENS_MSG_SENSORS 0x01 +#define WMR_MS_HOLOLENS_MSG_CONTROL 0x02 +#define WMR_MS_HOLOLENS_MSG_DEBUG 0x03 +#define WMR_MS_HOLOLENS_MSG_UNKNOWN_05 0x05 +#define WMR_MS_HOLOLENS_MSG_UNKNOWN_06 0x06 +#define WMR_MS_HOLOLENS_MSG_UNKNOWN_0E 0x0E +#define WMR_MS_HOLOLENS_MSG_UNKNOWN_17 0x17 + +#define WMR_CONTROL_MSG_IPD_VALUE 0x01 +#define WMR_CONTROL_MSG_UNKNOWN_05 0x05 + + +static const unsigned char hololens_sensors_imu_on[64] = {0x02, 0x07}; + + +struct hololens_sensors_packet +{ + uint8_t id; + uint16_t temperature[4]; + uint64_t gyro_timestamp[4]; + int16_t gyro[3][4 * 8]; + uint64_t accel_timestamp[4]; + int32_t accel[3][4]; + uint64_t video_timestamp[4]; +}; + +struct wmr_config_header +{ + uint32_t json_start; + uint32_t json_size; + char manufacturer[0x40]; + char device[0x40]; + char serial[0x40]; + char uid[0x26]; + char unk[0xd5]; + char name[0x40]; + char revision[0x20]; + char revision_date[0x20]; +}; + +/*! + * @} + */ + + +/*! + * WMR and MS HoloLens Sensors protocol helpers + * + * @ingroup drv_wmr + * @{ + */ + +void +vec3_from_hololens_accel(int32_t sample[3][4], int i, struct xrt_vec3 *out_vec); + +void +vec3_from_hololens_gyro(int16_t sample[3][32], int i, struct xrt_vec3 *out_vec); + + +static inline uint8_t +read8(const unsigned char **buffer) +{ + uint8_t ret = **buffer; + *buffer += 1; + return ret; +} + +static inline int16_t +read16(const unsigned char **buffer) +{ + int16_t ret = (*(*buffer + 0) << 0) | // + (*(*buffer + 1) << 8); + *buffer += 2; + return ret; +} + +static inline int32_t +read32(const unsigned char **buffer) +{ + int32_t ret = (*(*buffer + 0) << 0) | // + (*(*buffer + 1) << 8) | // + (*(*buffer + 2) << 16) | // + (*(*buffer + 3) << 24); + *buffer += 4; + return ret; +} + +static inline uint64_t +read64(const unsigned char **buffer) +{ + uint64_t ret = ((uint64_t) * (*buffer + 0) << 0) | // + ((uint64_t) * (*buffer + 1) << 8) | // + ((uint64_t) * (*buffer + 2) << 16) | // + ((uint64_t) * (*buffer + 3) << 24) | // + ((uint64_t) * (*buffer + 4) << 32) | // + ((uint64_t) * (*buffer + 5) << 40) | // + ((uint64_t) * (*buffer + 6) << 48) | // + ((uint64_t) * (*buffer + 7) << 56); + *buffer += 8; + return ret; +} + +/*! + * @} + */ + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/include/xrt/meson.build b/src/xrt/include/xrt/meson.build index edb000a8f..fdaa03180 100644 --- a/src/xrt/include/xrt/meson.build +++ b/src/xrt/include/xrt/meson.build @@ -48,7 +48,7 @@ if libuvc.found() have_conf.set('XRT_HAVE_LIBUVC', true) endif -if opencv.found() +if opencv.found() and build_tracking have_conf.set('XRT_HAVE_OPENCV', true) endif @@ -68,18 +68,18 @@ if has_v4l2_header and 'v4l2' in drivers have_conf.set('XRT_HAVE_V4L2', true) endif -if 'vf' in drivers - have_conf.set('XRT_HAVE_VF', true) -endif - if true - have_conf.set('XRT_HAVE_VULKAN', true) + have_conf.set('XRT_HAVE_VULKAN', true) endif if dbus.found() and not get_option('dbus').disabled() have_conf.set('XRT_HAVE_DBUS', true) endif +if systemd.found() and not get_option('systemd').disabled() + have_conf.set('XRT_HAVE_SYSTEMD', true) +endif + if get_option('layer_depth') have_conf.set('XRT_FEATURE_OPENXR_LAYER_DEPTH', true) endif @@ -100,6 +100,27 @@ if get_option('layer_equirect2') have_conf.set('XRT_FEATURE_OPENXR_LAYER_EQUIRECT2', true) endif +if libbsd.found() and not get_option('libbsd').disabled() + have_conf.set('XRT_HAVE_LIBBSD', true) +endif + +if slam.found() and build_tracking + have_conf.set('XRT_HAVE_SLAM', true) + have_conf.set('XRT_HAVE_KIMERA_SLAM', slam.name() == 'kimera_vio') +endif + +if build_wayland + have_conf.set('XRT_HAVE_WAYLAND', true) +endif + +if build_wayland_direct + have_conf.set('XRT_HAVE_WAYLAND_DIRECT', true) +endif + +if build_tracing + have_conf.set('XRT_HAVE_PERCETTO', true) +endif + xrt_config_have_h = configure_file( output: 'xrt_config_have.h', configuration: have_conf, @@ -120,6 +141,10 @@ if get_option('service') build_conf.set('XRT_FEATURE_SERVICE', true) endif +if get_option('tracing') + build_conf.set('XRT_FEATURE_TRACING', true) +endif + if get_option('color_log') build_conf.set('XRT_FEATURE_COLOR_LOG', true) endif diff --git a/src/xrt/include/xrt/xrt_compiler.h b/src/xrt/include/xrt/xrt_compiler.h index 19c97a4e5..43a22c42b 100644 --- a/src/xrt/include/xrt/xrt_compiler.h +++ b/src/xrt/include/xrt/xrt_compiler.h @@ -130,11 +130,9 @@ xrt_atomic_s32_cmpxchg(xrt_atomic_s32_t *p, int32_t old_, int32_t new_) } #ifdef _MSC_VER -#ifdef XRT_64_BIT -typedef int64_t ssize_t; -#else -typedef int32_t ssize_t; -#endif +typedef intptr_t ssize_t; +#define _SSIZE_T_ +#define _SSIZE_T_DEFINED #endif /*! diff --git a/src/xrt/include/xrt/xrt_compositor.h b/src/xrt/include/xrt/xrt_compositor.h index a39744736..bccfaedc6 100644 --- a/src/xrt/include/xrt/xrt_compositor.h +++ b/src/xrt/include/xrt/xrt_compositor.h @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -37,7 +37,7 @@ typedef uint64_t VkDeviceMemory; /*! - * @ingroup xrt_iface + * @addtogroup xrt_iface * @{ */ @@ -141,6 +141,8 @@ struct xrt_sub_image uint32_t array_index; //! The rectangle in the image to use struct xrt_rect rect; + //! Normalized sub image coordinates and size. + struct xrt_normalized_rect norm_rect; }; /*! @@ -362,6 +364,11 @@ struct xrt_layer_data */ struct xrt_swapchain { + /*! + * Reference helper. + */ + struct xrt_reference reference; + /*! * Number of images. * @@ -399,6 +406,39 @@ struct xrt_swapchain xrt_result_t (*release_image)(struct xrt_swapchain *xsc, uint32_t index); }; +/*! + * Update the reference counts on swapchain(s). + * + * @param dst Pointer to a object reference, if the object reference is + * non-null will decrement it's counter. The reference that + * @p dst points to will be set to @p src. + * @param[in] src Object to be have it's refcount increased @p dst is set to + * this. + * @ingroup xrt_iface + * @relates xrt_swapchain + */ +static inline void +xrt_swapchain_reference(struct xrt_swapchain **dst, struct xrt_swapchain *src) +{ + struct xrt_swapchain *old_dst = *dst; + + if (old_dst == src) { + return; + } + + if (src) { + xrt_reference_inc(&src->reference); + } + + *dst = src; + + if (old_dst) { + if (xrt_reference_dec(&old_dst->reference)) { + old_dst->destroy(old_dst); + } + } +} + /*! * @copydoc xrt_swapchain::acquire_image * @@ -438,26 +478,69 @@ xrt_swapchain_release_image(struct xrt_swapchain *xsc, uint32_t index) return xsc->release_image(xsc, index); } + +/* + * + * Fence. + * + */ + /*! - * @copydoc xrt_swapchain::destroy + * Compositor fence used for syncornization. + */ +struct xrt_compositor_fence +{ + /*! + * Destroys the fence. + */ + xrt_result_t (*wait)(struct xrt_compositor_fence *xcf, uint64_t timeout); + + /*! + * Destroys the fence. + */ + void (*destroy)(struct xrt_compositor_fence *xcf); +}; + +/*! + * @copydoc xrt_compositor_fence::wait + * + * Helper for calling through the function pointer. + * + * @public @memberof xrt_compositor_fence + */ +static inline xrt_result_t +xrt_compositor_fence_wait(struct xrt_compositor_fence *xcf, uint64_t timeout) +{ + return xcf->wait(xcf, timeout); +} + +/*! + * @copydoc xrt_compositor_fence::destroy * * Helper for calling through the function pointer: does a null check and sets - * xsc_ptr to null if freed. + * xcf_ptr to null if freed. * - * @public @memberof xrt_swapchain + * @public @memberof xrt_compositor_fence */ static inline void -xrt_swapchain_destroy(struct xrt_swapchain **xsc_ptr) +xrt_compositor_fence_destroy(struct xrt_compositor_fence **xcf_ptr) { - struct xrt_swapchain *xsc = *xsc_ptr; - if (xsc == NULL) { + struct xrt_compositor_fence *xcf = *xcf_ptr; + if (xcf == NULL) { return; } - xsc->destroy(xsc); - *xsc_ptr = NULL; + xcf->destroy(xcf); + *xcf_ptr = NULL; } + +/* + * + * Events. + * + */ + /*! * Event type for compositor events, none means no event was returned. */ @@ -496,6 +579,18 @@ union xrt_compositor_event { struct xrt_compositor_event_state_change overlay; }; + +/* + * + * Compositor. + * + */ + +enum xrt_compositor_frame_point +{ + XRT_COMPOSITOR_FRAME_POINT_WOKE, //!< The client woke up after waiting. +}; + /*! * Swapchain creation info. */ @@ -553,6 +648,10 @@ struct xrt_compositor /*! * Create a swapchain with a set of images. + * + * The pointer pointed to by @p out_xsc has to either be NULL or a valid + * @ref xrt_swapchain pointer. If there is a valid @ref xrt_swapchain + * pointed by the pointed pointer it will have it reference decremented. */ xrt_result_t (*create_swapchain)(struct xrt_compositor *xc, const struct xrt_swapchain_create_info *info, @@ -560,6 +659,10 @@ struct xrt_compositor /*! * Create a swapchain from a set of native images. + * + * The pointer pointed to by @p out_xsc has to either be NULL or a valid + * @ref xrt_swapchain pointer. If there is a valid @ref xrt_swapchain + * pointed by the pointed pointer it will have it reference decremented. */ xrt_result_t (*import_swapchain)(struct xrt_compositor *xc, const struct xrt_swapchain_create_info *info, @@ -567,6 +670,13 @@ struct xrt_compositor uint32_t num_images, struct xrt_swapchain **out_xsc); + /*! + * Create a compositor fence from a native sync handle. + */ + xrt_result_t (*import_fence)(struct xrt_compositor *xc, + xrt_graphics_sync_handle_t handle, + struct xrt_compositor_fence **out_xcf); + /*! * Poll events from this compositor. * @@ -586,6 +696,39 @@ struct xrt_compositor */ xrt_result_t (*end_session)(struct xrt_compositor *xc); + /*! + * This function and @ref mark_frame function calls are a alternative to + * @ref wait_frame. + * + * The only requirement on the compositor for the @p frame_id + * is that it is a positive number. + * + * @param[out] xc The compositor + * @param[out] out_frame_id Frame id + * @param[out] out_wake_time_ns When we want the client to be awoken to begin rendering. + * @param[out] out_predicted_gpu_time_ns When we expect the client to finish the GPU work. + * @param[out] out_predicted_display_time_ns When the pixels turns into photons. + * @param[out] out_predicted_display_period_ns The period for the frames. + */ + xrt_result_t (*predict_frame)(struct xrt_compositor *xc, + int64_t *out_frame_id, + uint64_t *out_wake_time_ns, + uint64_t *out_predicted_gpu_time_ns, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns); + + /*! + * This function and @ref predict_frame function calls are a alternative to + * @ref wait_frame. + * + * The client calls this function to mark that it woke up from waiting + * on a frame. + */ + xrt_result_t (*mark_frame)(struct xrt_compositor *xc, + int64_t frame_id, + enum xrt_compositor_frame_point point, + uint64_t when_ns); + /*! * See xrWaitFrame. * @@ -621,7 +764,10 @@ struct xrt_compositor * @p layer_commit that layers will be displayed. From the point of view * of the swapchain the image is used as soon as it's given in a call. */ - xrt_result_t (*layer_begin)(struct xrt_compositor *xc, int64_t frame_id, enum xrt_blend_mode env_blend_mode); + xrt_result_t (*layer_begin)(struct xrt_compositor *xc, + int64_t frame_id, + uint64_t display_time_ns, + enum xrt_blend_mode env_blend_mode); /*! * Adds a stereo projection layer for submissions. @@ -773,6 +919,21 @@ xrt_comp_import_swapchain(struct xrt_compositor *xc, return xc->import_swapchain(xc, info, native_images, num_images, out_xsc); } +/*! + * @copydoc xrt_compositor::import_fence + * + * Helper for calling through the function pointer. + * + * @public @memberof xrt_compositor + */ +static inline xrt_result_t +xrt_comp_import_fence(struct xrt_compositor *xc, + xrt_graphics_sync_handle_t handle, + struct xrt_compositor_fence **out_xcf) +{ + return xc->import_fence(xc, handle, out_xcf); +} + /*! * @copydoc xrt_compositor::poll_events * @@ -812,6 +973,46 @@ xrt_comp_end_session(struct xrt_compositor *xc) return xc->end_session(xc); } +/*! + * @copydoc xrt_compositor::predict_frame + * + * Helper for calling through the function pointer. + * + * @public @memberof xrt_compositor + */ +static inline xrt_result_t +xrt_comp_predict_frame(struct xrt_compositor *xc, + int64_t *out_frame_id, + uint64_t *out_wake_time_ns, + uint64_t *out_predicted_gpu_time_ns, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns) +{ + return xc->predict_frame( // + xc, // + out_frame_id, // + out_wake_time_ns, // + out_predicted_gpu_time_ns, // + out_predicted_display_time_ns, // + out_predicted_display_period_ns); // +} + +/*! + * @copydoc xrt_compositor::mark_frame + * + * Helper for calling through the function pointer. + * + * @public @memberof xrt_compositor + */ +static inline xrt_result_t +xrt_comp_mark_frame(struct xrt_compositor *xc, + int64_t frame_id, + enum xrt_compositor_frame_point point, + uint64_t when_ns) +{ + return xc->mark_frame(xc, frame_id, point, when_ns); +} + /*! * @copydoc xrt_compositor::wait_frame * @@ -862,9 +1063,12 @@ xrt_comp_discard_frame(struct xrt_compositor *xc, int64_t frame_id) * @public @memberof xrt_compositor */ static inline xrt_result_t -xrt_comp_layer_begin(struct xrt_compositor *xc, int64_t frame_id, enum xrt_blend_mode env_blend_mode) +xrt_comp_layer_begin(struct xrt_compositor *xc, + int64_t frame_id, + uint64_t display_time_ns, + enum xrt_blend_mode env_blend_mode) { - return xc->layer_begin(xc, frame_id, env_blend_mode); + return xc->layer_begin(xc, frame_id, display_time_ns, env_blend_mode); } /*! @@ -1158,6 +1362,7 @@ struct xrt_image_native * Native buffer handle. */ xrt_graphics_buffer_handle_t handle; + /*! * @brief Buffer size in memory. * @@ -1166,6 +1371,11 @@ struct xrt_image_native * into Vulkan. */ size_t size; + + /*! + * Is the image created with a deicated allocation or not. + */ + bool use_dedicated_allocation; }; /*! @@ -1184,6 +1394,17 @@ struct xrt_swapchain_native struct xrt_image_native images[XRT_MAX_SWAPCHAIN_IMAGES]; }; +/*! + * @copydoc xrt_swapchain_reference + * + * @relates xrt_swapchain_native + */ +static inline void +xrt_swapchain_native_reference(struct xrt_swapchain_native **dst, struct xrt_swapchain_native *src) +{ + xrt_swapchain_reference((struct xrt_swapchain **)dst, (struct xrt_swapchain *)src); +} + /*! * @interface xrt_compositor_native * @@ -1207,6 +1428,10 @@ struct xrt_compositor_native * Helper for calling through the base's function pointer then performing the * known-safe downcast. * + * The pointer pointed to by @p out_xsc has to either be NULL or a valid + * @ref xrt_swapchain pointer. If there is a valid @ref xrt_swapchain + * pointed by the pointed pointer it will have it reference decremented. + * * @public @memberof xrt_compositor_native */ static inline xrt_result_t @@ -1214,11 +1439,17 @@ xrt_comp_native_create_swapchain(struct xrt_compositor_native *xcn, const struct xrt_swapchain_create_info *info, struct xrt_swapchain_native **out_xscn) { - struct xrt_swapchain *xsc = NULL; + struct xrt_swapchain *xsc = NULL; // Has to be NULL. + xrt_result_t ret = xrt_comp_create_swapchain(&xcn->base, info, &xsc); if (ret == XRT_SUCCESS) { + // Need to dereference any swapchain already there first. + xrt_swapchain_native_reference(out_xscn, NULL); + + // Already referenced. *out_xscn = (struct xrt_swapchain_native *)xsc; } + return ret; } @@ -1287,6 +1518,39 @@ struct xrt_system_compositor_info uint8_t client_vk_deviceUUID[XRT_GPU_UUID_SIZE]; }; +struct xrt_system_compositor; + +/*! + * Special functions to control multi session/clients. + */ +struct xrt_multi_compositor_control +{ + /*! + * Sets the state of the compositor, generating any events to the client + * if the state is actually changed. Input focus is enforced/handled by + * a different component but is still signaled by the compositor. + */ + xrt_result_t (*set_state)(struct xrt_system_compositor *xsc, + struct xrt_compositor *xc, + bool visible, + bool focused); + + /*! + * Set the rendering Z order for rendering, visible has higher priority + * then z_order but is still saved until visible again. This a signed + * 64 bit integer compared to a unsigned 32 bit integer in OpenXR, so + * that non-overlay clients can be handled like overlay ones. + */ + xrt_result_t (*set_z_order)(struct xrt_system_compositor *xsc, struct xrt_compositor *xc, int64_t z_order); + + /*! + * Tell this client/session if the main application is visible or not. + */ + xrt_result_t (*set_main_app_visibility)(struct xrt_system_compositor *xsc, + struct xrt_compositor *xc, + bool visible); +}; + /*! * The system compositor is a long lived object, it has the same life time as a * XrSystemID. @@ -1296,6 +1560,11 @@ struct xrt_system_compositor //! Info regarding the system. struct xrt_system_compositor_info info; + /*! + * Does this system compositor support multi client controls. + */ + struct xrt_multi_compositor_control *xmcc; + /*! * Create a new native compositor. * @@ -1318,6 +1587,58 @@ struct xrt_system_compositor void (*destroy)(struct xrt_system_compositor *xsc); }; +/*! + * @copydoc xrt_multi_compositor_control::set_state + * + * Helper for calling through the function pointer. + * + * @public @memberof xrt_system_compositor + */ +static inline xrt_result_t +xrt_syscomp_set_state(struct xrt_system_compositor *xsc, struct xrt_compositor *xc, bool visible, bool focused) +{ + if (xsc->xmcc == NULL) { + return XRT_ERROR_MULTI_SESSION_NOT_IMPLEMENTED; + } + + return xsc->xmcc->set_state(xsc, xc, visible, focused); +} + +/*! + * @copydoc xrt_multi_compositor_control::set_z_order + * + * Helper for calling through the function pointer. + * + * @public @memberof xrt_system_compositor + */ +static inline xrt_result_t +xrt_syscomp_set_z_order(struct xrt_system_compositor *xsc, struct xrt_compositor *xc, int64_t z_order) +{ + if (xsc->xmcc == NULL) { + return XRT_ERROR_MULTI_SESSION_NOT_IMPLEMENTED; + } + + return xsc->xmcc->set_z_order(xsc, xc, z_order); +} + + +/*! + * @copydoc xrt_multi_compositor_control::set_main_app_visibility + * + * Helper for calling through the function pointer. + * + * @public @memberof xrt_system_compositor + */ +static inline xrt_result_t +xrt_syscomp_set_main_app_visibility(struct xrt_system_compositor *xsc, struct xrt_compositor *xc, bool visible) +{ + if (xsc->xmcc == NULL) { + return XRT_ERROR_MULTI_SESSION_NOT_IMPLEMENTED; + } + + return xsc->xmcc->set_main_app_visibility(xsc, xc, visible); +} + /*! * @copydoc xrt_system_compositor::create_native_compositor * diff --git a/src/xrt/include/xrt/xrt_config_build.h.cmake_in b/src/xrt/include/xrt/xrt_config_build.h.cmake_in index 94f40d217..a6fc7e0ab 100644 --- a/src/xrt/include/xrt/xrt_config_build.h.cmake_in +++ b/src/xrt/include/xrt/xrt_config_build.h.cmake_in @@ -26,3 +26,5 @@ #cmakedefine XRT_FEATURE_OPENXR_LAYER_EQUIRECT1 #cmakedefine XRT_FEATURE_COLOR_LOG + +#cmakedefine XRT_FEATURE_TRACING diff --git a/src/xrt/include/xrt/xrt_config_have.h.cmake_in b/src/xrt/include/xrt/xrt_config_have.h.cmake_in index fa3637f68..06b7cd521 100644 --- a/src/xrt/include/xrt/xrt_config_have.h.cmake_in +++ b/src/xrt/include/xrt/xrt_config_have.h.cmake_in @@ -10,9 +10,10 @@ #pragma once #cmakedefine XRT_HAVE_DBUS -#cmakedefine XRT_HAVE_VF +#cmakedefine XRT_HAVE_LIBBSD #cmakedefine XRT_HAVE_EGL #cmakedefine XRT_HAVE_FFMPEG +#cmakedefine XRT_HAVE_GST #cmakedefine XRT_HAVE_JPEG #cmakedefine XRT_HAVE_LIBUDEV #cmakedefine XRT_HAVE_LIBUSB @@ -24,3 +25,8 @@ #cmakedefine XRT_HAVE_SYSTEMD #cmakedefine XRT_HAVE_V4L2 #cmakedefine XRT_HAVE_VULKAN +#cmakedefine XRT_HAVE_PERCETTO +#cmakedefine XRT_HAVE_KIMERA_SLAM +#cmakedefine XRT_HAVE_SLAM +#cmakedefine XRT_HAVE_WAYLAND +#cmakedefine XRT_HAVE_WAYLAND_DIRECT diff --git a/src/xrt/include/xrt/xrt_defines.h b/src/xrt/include/xrt/xrt_defines.h index 04d8cf6c8..ab7b0734e 100644 --- a/src/xrt/include/xrt/xrt_defines.h +++ b/src/xrt/include/xrt/xrt_defines.h @@ -1,9 +1,10 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Common defines and enums for XRT. * @author Jakob Bornecrantz + * @author Moses Turner * @ingroup xrt_iface */ @@ -32,19 +33,20 @@ struct xrt_reference }; /*! - * Which blend mode does the device support, used as both a bitfield and value. + * Blend mode that the device supports, exact mirror of XrEnvironmentBlendMode * * @ingroup xrt_iface */ enum xrt_blend_mode { - // clang-format off - XRT_BLEND_MODE_OPAQUE = 1 << 0, - XRT_BLEND_MODE_ADDITIVE = 1 << 1, - XRT_BLEND_MODE_ALPHA_BLEND = 1 << 2, - // clang-format on + XRT_BLEND_MODE_OPAQUE = 1, + XRT_BLEND_MODE_ADDITIVE = 2, + XRT_BLEND_MODE_ALPHA_BLEND = 3, + XRT_BLEND_MODE_MAX_ENUM, }; +#define XRT_MAX_DEVICE_BLEND_MODES 3 + /*! * Which distortion model does the device expose, * used both as a bitfield and value. @@ -109,6 +111,17 @@ struct xrt_quat float w; }; +/*! + * Identity value for @ref xrt_quat + * + * @ingroup xrt_iface math + * @relates xrt_quat + */ +#define XRT_QUAT_IDENTITY \ + { \ + 0.f, 0.f, 0.f, 1.f \ + } + /*! * A 1 element vector with single floats. * @@ -152,6 +165,59 @@ struct xrt_vec3 float z; }; +/*! + * A 3 element vector with single doubles. + * + * @ingroup xrt_iface math + */ +struct xrt_vec3_f64 +{ + double x; + double y; + double z; +}; + +/*! + * All-zero value for @ref xrt_vec3 + * + * @ingroup xrt_iface math + * @relates xrt_vec3 + */ +#define XRT_VEC3_ZERO \ + { \ + 0.f, 0.f, 0.f \ + } +/*! + * Value for @ref xrt_vec3 with 1 in the @p x coordinate. + * + * @ingroup xrt_iface math + * @relates xrt_vec3 + */ +#define XRT_VEC3_UNIT_X \ + { \ + 1.f, 0.f, 0.f \ + } +/*! + * Value for @ref xrt_vec3 with 1 in the @p y coordinate. + * + * @ingroup xrt_iface math + * @relates xrt_vec3 + */ +#define XRT_VEC3_UNIT_Y \ + { \ + 0.f, 1.f, 0.f \ + } +/*! + * Value for @ref xrt_vec3 with 1 in the @p z coordinate. + * + * @ingroup xrt_iface math + * @relates xrt_vec3 + */ +#define XRT_VEC3_UNIT_Z \ + { \ + 0.f, 0.f, 1.f \ + } + /*! * A 3 element vector with 32 bit integers. * @@ -257,6 +323,16 @@ struct xrt_rect struct xrt_size extent; }; +/*! + * Normalized image rectangle, coordinates and size in 0 .. 1 range. + * + * @ingroup xrt_iface math + */ +struct xrt_normalized_rect +{ + float x, y, w, h; +}; + /*! * A pose composed of a position and orientation. * @@ -269,6 +345,16 @@ struct xrt_pose struct xrt_quat orientation; struct xrt_vec3 position; }; +/*! + * Identity value for @ref xrt_pose + * + * @ingroup xrt_iface math + * @relates xrt_pose + */ +#define XRT_POSE_IDENTITY \ + { \ + XRT_QUAT_IDENTITY, XRT_VEC3_ZERO \ + } /*! * Describes a projection matrix fov. @@ -316,6 +402,16 @@ struct xrt_matrix_4x4 float v[16]; }; +/*! + * A tightly packed 4x4 matrix of double. + * + * @ingroup xrt_iface math + */ +struct xrt_matrix_4x4_f64 +{ + double v[16]; +}; + /*! * A range of API versions supported. * @@ -372,6 +468,20 @@ struct xrt_space_relation struct xrt_vec3 angular_velocity; }; +/*! + * A zero/identity value for @ref xrt_space_relation + * + * @note Despite this initializing all members (to zero or identity), this sets the xrt_space_relation::relation_flags + * to XRT_SPACE_RELATION_BITMASK_NONE - so this is safe to assign before an error return, etc. + * + * @ingroup xrt_iface math + * @relates xrt_space_relation + */ +#define XRT_SPACE_RELATION_ZERO \ + { \ + XRT_SPACE_RELATION_BITMASK_NONE, XRT_POSE_IDENTITY, XRT_VEC3_ZERO, XRT_VEC3_ZERO \ + } + /*! * The maximum number of steps that can be in a space graph chain. * @@ -428,6 +538,8 @@ enum xrt_device_name XRT_DEVICE_HAND_INTERACTION, XRT_DEVICE_HAND_TRACKER, + + XRT_DEVICE_REALSENSE, }; /*! @@ -508,6 +620,7 @@ enum xrt_input_name XRT_INPUT_GENERIC_HEAD_DETECT = XRT_INPUT_NAME(0x0001, BOOLEAN), XRT_INPUT_GENERIC_HAND_TRACKING_LEFT = XRT_INPUT_NAME(0x0002, HAND_TRACKING), XRT_INPUT_GENERIC_HAND_TRACKING_RIGHT = XRT_INPUT_NAME(0x0004, HAND_TRACKING), + XRT_INPUT_GENERIC_TRACKER_POSE = XRT_INPUT_NAME(0x0005, POSE), XRT_INPUT_SIMPLE_SELECT_CLICK = XRT_INPUT_NAME(0x0010, BOOLEAN), XRT_INPUT_SIMPLE_MENU_CLICK = XRT_INPUT_NAME(0x0011, BOOLEAN), @@ -751,6 +864,7 @@ struct xrt_hand_joint_set // in driver global space, without tracking_origin offset struct xrt_space_relation hand_pose; + bool is_active; }; /*! @@ -795,7 +909,7 @@ enum xrt_output_name XRT_OUTPUT_NAME_WMR_HAPTIC = XRT_OUTPUT_NAME(0x0050, VIBRATION), XRT_OUTPUT_NAME_XBOX_HAPTIC_LEFT = XRT_OUTPUT_NAME(0x0060, VIBRATION), - XRT_OUTPUT_NAME_XBOX_HAPTIC_RIGHTT = XRT_OUTPUT_NAME(0x0061, VIBRATION), + XRT_OUTPUT_NAME_XBOX_HAPTIC_RIGHT = XRT_OUTPUT_NAME(0x0061, VIBRATION), XRT_OUTPUT_NAME_XBOX_HAPTIC_LEFT_TRIGGER = XRT_OUTPUT_NAME(0x0062, VIBRATION), XRT_OUTPUT_NAME_XBOX_HAPTIC_RIGHT_TRIGGER = XRT_OUTPUT_NAME(0x0063, VIBRATION), diff --git a/src/xrt/include/xrt/xrt_device.h b/src/xrt/include/xrt/xrt_device.h index 47ea18d7e..a5ca5d4fb 100644 --- a/src/xrt/include/xrt/xrt_device.h +++ b/src/xrt/include/xrt/xrt_device.h @@ -4,6 +4,7 @@ * @file * @brief Header defining an xrt HMD device. * @author Jakob Bornecrantz + * @author Moses Turner * @ingroup xrt_iface */ @@ -106,9 +107,10 @@ struct xrt_hmd_parts struct xrt_view views[2]; /*! - * Supported blend modes, a bitfield. + * Array of supported blend modes. */ - enum xrt_blend_mode blend_mode; + enum xrt_blend_mode blend_modes[XRT_MAX_DEVICE_BLEND_MODES]; + size_t num_blend_modes; /*! * Distortion information. @@ -125,20 +127,20 @@ struct xrt_hmd_parts //! Data. float *vertices; //! Number of vertices. - size_t num_vertices; + uint32_t num_vertices; //! Stride of vertices - size_t stride; + uint32_t stride; //! 1 or 3 for (chromatic aberration). - size_t num_uv_channels; + uint32_t num_uv_channels; //! Indices, for triangle strip. int *indices; //! Number of indices for the triangle strip. - size_t num_indices[2]; + uint32_t num_indices[2]; //! Offsets for the indices. - size_t offset_indices[2]; + uint32_t offset_indices[2]; //! Total number of indices. - size_t total_num_indices; + uint32_t total_num_indices; } mesh; } distortion; }; @@ -226,6 +228,9 @@ struct xrt_device //! A string describing the device. char str[XRT_DEVICE_NAME_LEN]; + //! A unique identifier. Persistent across configurations, if possible. + char serial[XRT_DEVICE_NAME_LEN]; + //! Null if this device does not interface with the users head. struct xrt_hmd_parts *hmd; @@ -284,8 +289,8 @@ struct xrt_device /*! * Get relationship of hand joints to the tracking origin space as - * the base space. It is the responsibility of the device driver to do - * any prediction, there are helper functions available for this. + * the base space. It is the responsibility of the device driver to either do prediction or return joints from a + * previous time and write that time out to out_timestamp_ns. * * The timestamps are system monotonic timestamps, such as returned by * os_monotonic_get_ns(). @@ -300,12 +305,16 @@ struct xrt_device * wants the pose to be from. * @param[out] out_relation The relation read from the device. * + * @param[out] out_timestamp_ns + * * @see xrt_input_name */ + void (*get_hand_tracking)(struct xrt_device *xdev, enum xrt_input_name name, - uint64_t at_timestamp_ns, - struct xrt_hand_joint_set *out_value); + uint64_t desired_timestamp_ns, + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp_ns); /*! * Set a output value. * @@ -336,7 +345,7 @@ struct xrt_device * orientation unless you have canted screens. */ void (*get_view_pose)(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose); @@ -382,9 +391,10 @@ static inline void xrt_device_get_hand_tracking(struct xrt_device *xdev, enum xrt_input_name name, uint64_t requested_timestamp_ns, - struct xrt_hand_joint_set *out_value) + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp_ns) { - xdev->get_hand_tracking(xdev, name, requested_timestamp_ns, out_value); + xdev->get_hand_tracking(xdev, name, requested_timestamp_ns, out_value, out_timestamp_ns); } /*! @@ -405,13 +415,24 @@ xrt_device_set_output(struct xrt_device *xdev, enum xrt_output_name name, union */ static inline void xrt_device_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { xdev->get_view_pose(xdev, eye_relation, view_index, out_pose); } +/*! + * Helper function for @ref xrt_device::compute_distortion. + * + * @public @memberof xrt_device + */ +static inline void +xrt_device_compute_distortion(struct xrt_device *xdev, int view, float u, float v, struct xrt_uv_triplet *result) +{ + xdev->compute_distortion(xdev, view, u, v, result); +} + /*! * Helper function for @ref xrt_device::destroy. * diff --git a/src/xrt/include/xrt/xrt_frame.h b/src/xrt/include/xrt/xrt_frame.h index d1bdc6861..707a28cd3 100644 --- a/src/xrt/include/xrt/xrt_frame.h +++ b/src/xrt/include/xrt/xrt_frame.h @@ -62,6 +62,19 @@ struct xrt_frame_sink void (*push_frame)(struct xrt_frame_sink *sink, struct xrt_frame *frame); }; +/*! + * @copydoc xrt_frame_sink::push_frame + * + * Helper for calling through the function pointer. + * + * @public @memberof xrt_frame_sink + */ +static inline void +xrt_sink_push_frame(struct xrt_frame_sink *sink, struct xrt_frame *frame) +{ + sink->push_frame(sink, frame); +} + /*! * @interface xrt_frame_node * diff --git a/src/xrt/include/xrt/xrt_frameserver.h b/src/xrt/include/xrt/xrt_frameserver.h index 2597f8ede..16c4a3bc7 100644 --- a/src/xrt/include/xrt/xrt_frameserver.h +++ b/src/xrt/include/xrt/xrt_frameserver.h @@ -17,6 +17,7 @@ extern "C" { #endif +struct xrt_slam_sinks; /*! * Controlling the camera capture parameters @@ -100,6 +101,18 @@ struct xrt_fs enum xrt_fs_capture_type capture_type, uint32_t descriptor_index); + /*! + * Setup SLAM sinks for all the sensors a SLAM implementation may supports and + * start the frame server stream. Use @ref xrt_fs::stream_start instead if + * you only need the image stream. + * + * @note Having this method extends the scope of the frameserver to something + * more akin to a generic data source instead of just serving frames. + * + * @todo Fix this incongruence. Maybe rename the interface to xrt_data_source. + */ + bool (*slam_stream_start)(struct xrt_fs *xfs, struct xrt_slam_sinks *sinks); + /*! * Stop the capture stream. */ @@ -160,6 +173,19 @@ xrt_fs_stream_start(struct xrt_fs *xfs, return xfs->stream_start(xfs, xs, capture_type, descriptor_index); } +/*! + * @copydoc xrt_fs::slam_stream_start + * + * Helper for calling through the function pointer. + * + * @public @memberof xrt_fs + */ +static inline bool +xrt_fs_slam_stream_start(struct xrt_fs *xfs, struct xrt_slam_sinks *sinks) +{ + return xfs->slam_stream_start(xfs, sinks); +} + /*! * @copydoc xrt_fs::stream_stop * diff --git a/src/xrt/include/xrt/xrt_gfx_egl.h b/src/xrt/include/xrt/xrt_gfx_egl.h index 44af50142..10c3b0735 100644 --- a/src/xrt/include/xrt/xrt_gfx_egl.h +++ b/src/xrt/include/xrt/xrt_gfx_egl.h @@ -29,12 +29,13 @@ struct time_state; * @ingroup xrt_iface * @public @memberof xrt_compositor_native */ -struct xrt_compositor_gl * +xrt_result_t xrt_gfx_provider_create_gl_egl(struct xrt_compositor_native *xcn, EGLDisplay display, EGLConfig config, EGLContext context, - PFNEGLGETPROCADDRESSPROC getProcAddress); + PFNEGLGETPROCADDRESSPROC getProcAddress, + struct xrt_compositor_gl **out_xcgl); #ifdef __cplusplus } diff --git a/src/xrt/include/xrt/xrt_instance.h b/src/xrt/include/xrt/xrt_instance.h index 98172d71a..256d4692f 100644 --- a/src/xrt/include/xrt/xrt_instance.h +++ b/src/xrt/include/xrt/xrt_instance.h @@ -25,7 +25,7 @@ struct xrt_system_compositor; /*! - * @ingroup xrt_iface + * @addtogroup xrt_iface * @{ */ @@ -52,7 +52,7 @@ struct xrt_instance_info * Each "target" will provide its own (private) implementation of this * interface, which is exposed by implementing xrt_instance_create(). * - * Additional information can be found in @ref md_targets + * Additional information can be found in @ref understanding-targets. * * @sa ipc_instance_create */ diff --git a/src/xrt/include/xrt/xrt_openxr_includes.h b/src/xrt/include/xrt/xrt_openxr_includes.h index 9ab416bc6..8db5dcfa8 100644 --- a/src/xrt/include/xrt/xrt_openxr_includes.h +++ b/src/xrt/include/xrt/xrt_openxr_includes.h @@ -33,6 +33,7 @@ typedef void *GLXContext; typedef void *EGLDisplay; typedef void *EGLContext; typedef void *EGLConfig; +typedef unsigned int EGLenum; typedef void (*__eglMustCastToProperFunctionPointerType)(void); typedef __eglMustCastToProperFunctionPointerType (*PFNEGLGETPROCADDRESSPROC)(const char *procname); #endif diff --git a/src/xrt/include/xrt/xrt_prober.h b/src/xrt/include/xrt/xrt_prober.h index ad7623bba..ef1272b87 100644 --- a/src/xrt/include/xrt/xrt_prober.h +++ b/src/xrt/include/xrt/xrt_prober.h @@ -1,4 +1,4 @@ -// Copyright 2019, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -33,13 +33,17 @@ struct xrt_tracking_factory; struct os_hid_device; /*! - * The maximum number of devices that a single "found" function called by the - * prober can create per-call. + * The maximum number of devices that a single + * @ref xrt_prober_entry::found or + * @ref xrt_auto_prober::lelo_dallas_autoprobe + * function called by the prober can create per-call. * * @ingroup xrt_iface */ #define XRT_MAX_DEVICES_PER_PROBE 16 +#define MAX_AUTO_PROBERS 16 + /*! * Entry for a single device. * @@ -58,6 +62,7 @@ struct xrt_prober_entry struct xrt_device **out_xdevs); const char *name; + const char *driver_name; }; /*! @@ -163,7 +168,15 @@ struct xrt_prober * xrt_prober_probe() */ int (*probe)(struct xrt_prober *xp); + + /*! + * Dump a listing of all devices found on the system to platform + * dependent output (stdout). + * + * @note Code consuming this interface should use xrt_prober_dump() + */ int (*dump)(struct xrt_prober *xp); + /*! * Iterate through drivers (by ID and auto-probers) checking to see if * they can handle any connected devices from the last xrt_prober::probe @@ -191,21 +204,49 @@ struct xrt_prober * and xrt_prober_select(). */ int (*select)(struct xrt_prober *xp, struct xrt_device **xdevs, size_t num_xdevs); + int (*open_hid_interface)(struct xrt_prober *xp, struct xrt_prober_device *xpdev, int interface, struct os_hid_device **out_hid_dev); + + /*! + * Opens the selected video device and returns a @ref xrt_fs, does not + * start it. + */ int (*open_video_device)(struct xrt_prober *xp, struct xrt_prober_device *xpdev, struct xrt_frame_context *xfctx, struct xrt_fs **out_xfs); + int (*list_video_devices)(struct xrt_prober *xp, xrt_prober_list_video_cb cb, void *ptr); + + int (*get_entries)(struct xrt_prober *xp, + size_t *out_num_entries, + struct xrt_prober_entry ***out_entries, + struct xrt_auto_prober ***out_auto_probers); + + /*! + * Returns a string property on the device of the given type + * @p which_string in @p out_buffer. + * + * @param[in] xp Prober. + * @param[in] xpdev Device to get string property from. + * @param[in] which_string Which string property to query. + * @param[in,out] out_buffer Target buffer. + * @param[in] max_length Max length of the target buffer. + * + * @return The length of the string, or negative on error. + * + */ int (*get_string_descriptor)(struct xrt_prober *xp, struct xrt_prober_device *xpdev, enum xrt_prober_string which_string, - unsigned char *buffer, - int length); + unsigned char *out_buffer, + size_t max_length); + bool (*can_open)(struct xrt_prober *xp, struct xrt_prober_device *xpdev); + /*! * Destroy the prober and set the pointer to null. * @@ -217,6 +258,8 @@ struct xrt_prober }; /*! + * @copydoc xrt_prober::probe + * * Helper function for @ref xrt_prober::probe. * * @public @memberof xrt_prober @@ -228,6 +271,8 @@ xrt_prober_probe(struct xrt_prober *xp) } /*! + * @copydoc xrt_prober::dump + * * Helper function for @ref xrt_prober::dump. * * @public @memberof xrt_prober @@ -239,6 +284,8 @@ xrt_prober_dump(struct xrt_prober *xp) } /*! + * @copydoc xrt_prober::select + * * Helper function for @ref xrt_prober::select. * * @public @memberof xrt_prober @@ -250,6 +297,8 @@ xrt_prober_select(struct xrt_prober *xp, struct xrt_device **xdevs, size_t num_x } /*! + * @copydoc xrt_prober::open_hid_interface + * * Helper function for @ref xrt_prober::open_hid_interface. * * @public @memberof xrt_prober @@ -264,6 +313,8 @@ xrt_prober_open_hid_interface(struct xrt_prober *xp, } /*! + * @copydoc xrt_prober::get_string_descriptor + * * Helper function for @ref xrt_prober::get_string_descriptor. * * @public @memberof xrt_prober @@ -272,13 +323,15 @@ static inline int xrt_prober_get_string_descriptor(struct xrt_prober *xp, struct xrt_prober_device *xpdev, enum xrt_prober_string which_string, - unsigned char *buffer, - int length) + unsigned char *out_buffer, + size_t max_length) { - return xp->get_string_descriptor(xp, xpdev, which_string, buffer, length); + return xp->get_string_descriptor(xp, xpdev, which_string, out_buffer, max_length); } /*! + * @copydoc xrt_prober::can_open + * * Helper function for @ref xrt_prober::can_open. * * @public @memberof xrt_prober @@ -291,6 +344,8 @@ xrt_prober_can_open(struct xrt_prober *xp, struct xrt_prober_device *xpdev) /*! + * @copydoc xrt_prober::open_video_device + * * Helper function for @ref xrt_prober::xrt_prober_open_video_device. * * @public @memberof xrt_prober @@ -305,6 +360,8 @@ xrt_prober_open_video_device(struct xrt_prober *xp, } /*! + * @copydoc xrt_prober::list_video_devices + * * Helper function for @ref xrt_prober::list_video_devices. * * @public @memberof xrt_prober @@ -316,7 +373,26 @@ xrt_prober_list_video_devices(struct xrt_prober *xp, xrt_prober_list_video_cb cb } /*! - * Helper function for @ref xrt_prober::destroy. + * @copydoc xrt_prober::get_entries + * + * Helper function for @ref xrt_prober::get_entries. + * + * @public @memberof xrt_prober + */ +static inline int +xrt_prober_get_entries(struct xrt_prober *xp, + size_t *out_num_entries, + struct xrt_prober_entry ***out_entries, + struct xrt_auto_prober ***out_auto_probers) +{ + return xp->get_entries(xp, out_num_entries, out_entries, out_auto_probers); +} + +/*! + * @copydoc xrt_prober::destroy + * + * Helper for calling through the function pointer: does a null check and sets + * xp_ptr to null if freed. * * @public @memberof xrt_prober */ @@ -329,6 +405,7 @@ xrt_prober_destroy(struct xrt_prober **xp_ptr) } xp->destroy(xp_ptr); + *xp_ptr = NULL; } /*! @@ -385,13 +462,19 @@ struct xrt_auto_prober * devices. * @param[in] xp Prober: provided for access to the tracking factory, * among other reasons. + * @param[out] out_xdevs Array of @ref XRT_MAX_DEVICES_PER_PROBE @c NULL + * @ref xrt_device pointers. First elements will be populated with new + * devices. * - * @return New device implementing the xrt_device interface, or NULL. + * @return The amount of devices written into @p out_xdevs, 0 if none. + * + * @note Leeloo Dallas is a reference to The Fifth Element. */ - struct xrt_device *(*lelo_dallas_autoprobe)(struct xrt_auto_prober *xap, - cJSON *attached_data, - bool no_hmds, - struct xrt_prober *xp); + int (*lelo_dallas_autoprobe)(struct xrt_auto_prober *xap, + cJSON *attached_data, + bool no_hmds, + struct xrt_prober *xp, + struct xrt_device **out_xdevs); /*! * Destroy this auto-prober. * diff --git a/src/xrt/include/xrt/xrt_results.h b/src/xrt/include/xrt/xrt_results.h index a178af9f3..9ddfb68d2 100644 --- a/src/xrt/include/xrt/xrt_results.h +++ b/src/xrt/include/xrt/xrt_results.h @@ -48,9 +48,26 @@ typedef enum xrt_result * Multiple not supported on this layer level (IPC, compositor). */ XRT_ERROR_MULTI_SESSION_NOT_IMPLEMENTED = -11, - /*! * The requested format is not supported by Monado. */ XRT_ERROR_SWAPCHAIN_FORMAT_UNSUPPORTED = -12, + /*! + * The given config was EGL_NO_CONFIG_KHR and EGL_KHR_no_config_context + * is not supported by the display. + */ + XRT_ERROR_EGL_CONFIG_MISSING = -13, + /*! + * Failed to initialize threading components. + */ + XRT_ERROR_THREADING_INIT_FAILURE = -14, + /*! + * The client has not created a session on this IPC connection, + * which is needed for the given command. + */ + XRT_ERROR_IPC_SESSION_NOT_CREATED = -15, + /*! + * The client has already created a session on this IPC connection. + */ + XRT_ERROR_IPC_SESSION_ALREADY_CREATED = -16, } xrt_result_t; diff --git a/src/xrt/include/xrt/xrt_settings.h b/src/xrt/include/xrt/xrt_settings.h index e8ff40df3..420153d40 100644 --- a/src/xrt/include/xrt/xrt_settings.h +++ b/src/xrt/include/xrt/xrt_settings.h @@ -12,13 +12,16 @@ #include "xrt/xrt_compiler.h" +// XRT_DEVICE_NAME_LEN +#include "xrt/xrt_device.h" + #ifdef __cplusplus extern "C" { #endif /*! - * @ingroup xrt_iface + * @addtogroup xrt_iface * @{ */ @@ -28,7 +31,7 @@ extern "C" { enum xrt_settings_camera_type { XRT_SETTINGS_CAMERA_TYPE_REGULAR_MONO = 0, - XRT_SETTINGS_CAMERA_TYPE_REGULAR_SBS = 1, + XRT_SETTINGS_CAMERA_TYPE_REGULAR_SBS = 1, // side-by-side XRT_SETTINGS_CAMERA_TYPE_PS4 = 2, XRT_SETTINGS_CAMERA_TYPE_LEAP_MOTION = 3, }; @@ -36,6 +39,23 @@ enum xrt_settings_camera_type #define XRT_SETTINGS_CAMERA_NAME_LENGTH 256 #define XRT_SETTINGS_PATH_LENGTH 1024 +#define XRT_MAX_TRACKING_OVERRIDES 16 + +enum xrt_tracking_override_type +{ + XRT_TRACKING_OVERRIDE_DIRECT = 0, + XRT_TRACKING_OVERRIDE_ATTACHED, +}; + +struct xrt_tracking_override +{ + char target_device_serial[XRT_DEVICE_NAME_LEN]; + char tracker_device_serial[XRT_DEVICE_NAME_LEN]; + enum xrt_input_name input_name; + struct xrt_pose offset; + enum xrt_tracking_override_type override_type; +}; + /*! * Holding enough information to recreate a tracking pipeline. */ diff --git a/src/xrt/include/xrt/xrt_tracking.h b/src/xrt/include/xrt/xrt_tracking.h index ee1e11d01..2887eef4c 100644 --- a/src/xrt/include/xrt/xrt_tracking.h +++ b/src/xrt/include/xrt/xrt_tracking.h @@ -27,12 +27,13 @@ struct xrt_tracking_factory; struct xrt_tracked_psmv; struct xrt_tracked_psvr; struct xrt_tracked_hand; +struct xrt_tracked_slam; //! @todo This is from u_time, duplicated to avoid layer violation. typedef int64_t timepoint_ns; /*! - * @ingroup xrt_iface + * @addtogroup xrt_iface * @{ */ @@ -57,6 +58,9 @@ enum xrt_tracking_type // The device(s) are tracked by external SLAM XRT_TRACKING_TYPE_EXTERNAL_SLAM, + + // The device(s) are tracked by other methods. + XRT_TRACKING_TYPE_OTHER, }; /*! @@ -108,10 +112,18 @@ struct xrt_tracking_factory int (*create_tracked_hand)(struct xrt_tracking_factory *, struct xrt_device *xdev, struct xrt_tracked_hand **out_hand); + + /*! + * Create a SLAM tracker. + */ + int (*create_tracked_slam)(struct xrt_tracking_factory *, + struct xrt_device *xdev, + struct xrt_tracked_slam **out_slam); }; /*! * IMU Sample. + * @todo Replace with @ref xrt_imu_sample */ struct xrt_tracking_sample { @@ -119,6 +131,46 @@ struct xrt_tracking_sample struct xrt_vec3 gyro_rad_secs; }; +/*! + * IMU Sample. + * @todo Make @ref xrt_tracked_psmv and @ref xrt_tracked_psvr use this + */ +struct xrt_imu_sample +{ + timepoint_ns timestamp_ns; + struct xrt_vec3_f64 accel_m_s2; + struct xrt_vec3_f64 gyro_rad_secs; +}; + +/*! + * @interface xrt_imu_sink + * + * An object that is sent IMU samples. + * + * Similar to @ref xrt_frame_sink but the interface implementation must manage + * its own resources, not through a context graph. + * + * @todo Make @ref xrt_tracked_psmv and @ref xrt_tracked_psvr implement this + */ +struct xrt_imu_sink +{ + /*! + * Push an IMU sample into the sink + */ + void (*push_imu)(struct xrt_imu_sink *, struct xrt_imu_sample *sample); +}; + +/*! + * Container of pointers to sinks that could be used for a SLAM system. Sinks + * are considered disabled if they are null. + */ +struct xrt_slam_sinks +{ + struct xrt_frame_sink *left; + struct xrt_frame_sink *right; + struct xrt_imu_sink *imu; +}; + /*! * @interface xrt_tracked_psmv * @@ -224,12 +276,37 @@ struct xrt_tracked_hand void (*destroy)(struct xrt_tracked_hand *); }; +/*! + * @interface xrt_tracked_slam + * + * An adapter that wraps an external SLAM tracker to provide SLAM tracking. + * Devices that want to be tracked through SLAM should create and manage an + * instance of this type. + */ +struct xrt_tracked_slam +{ + /*! + * Called by the owning @ref xrt_device to get the last estimated pose + * of the SLAM tracker. + */ + void (*get_tracked_pose)(struct xrt_tracked_slam *, + timepoint_ns when_ns, + struct xrt_space_relation *out_relation); +}; + /* * * Helper functions. * */ +//! @public @memberof xrt_imu_sink +static inline void +xrt_sink_push_imu(struct xrt_imu_sink *sink, struct xrt_imu_sample *sample) +{ + sink->push_imu(sink, sample); +} + //! @public @memberof xrt_tracked_psmv static inline void xrt_tracked_psmv_get_tracked_pose(struct xrt_tracked_psmv *psmv, @@ -301,6 +378,15 @@ xrt_tracked_hand_get_joints(struct xrt_tracked_hand *h, h->get_tracked_joints(h, name, when_ns, out_joints, out_relation); } +//! @public @memberof xrt_tracked_slam +static inline void +xrt_tracked_slam_get_tracked_pose(struct xrt_tracked_slam *slam, + timepoint_ns when_ns, + struct xrt_space_relation *out_relation) +{ + slam->get_tracked_pose(slam, when_ns, out_relation); +} + /*! * @} */ diff --git a/src/xrt/ipc/CMakeLists.txt b/src/xrt/ipc/CMakeLists.txt index cfc472d1c..1d4c9e62c 100644 --- a/src/xrt/ipc/CMakeLists.txt +++ b/src/xrt/ipc/CMakeLists.txt @@ -110,10 +110,9 @@ if(ANDROID) xrt-external-jnipp aux_android ) - set_target_properties(ipc_android - PROPERTIES - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON) + target_sources(ipc_server PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/server/ipc_server_mainloop_android.c + ) target_link_libraries(ipc_server PUBLIC ${ANDROID_LIBRARY} PRIVATE @@ -126,4 +125,8 @@ if(ANDROID) aux_android ipc_android ) +elseif(XRT_HAVE_LINUX) + target_sources(ipc_server PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/server/ipc_server_mainloop_linux.c + ) endif() diff --git a/src/xrt/ipc/android/build.gradle b/src/xrt/ipc/android/build.gradle index 85ba15ea6..c1ec82a5e 100644 --- a/src/xrt/ipc/android/build.gradle +++ b/src/xrt/ipc/android/build.gradle @@ -11,7 +11,7 @@ plugins { android { compileSdkVersion project.sharedTargetSdk - buildToolsVersion '30.0.2' + buildToolsVersion '30.0.3' defaultConfig { diff --git a/src/xrt/ipc/android/ipc_client_android.cpp b/src/xrt/ipc/android/ipc_client_android.cpp index 46adb195a..3d69a346f 100644 --- a/src/xrt/ipc/android/ipc_client_android.cpp +++ b/src/xrt/ipc/android/ipc_client_android.cpp @@ -19,6 +19,8 @@ using wrap::android::app::Activity; using wrap::org::freedesktop::monado::ipc::Client; +using xrt::auxiliary::android::getAppInfo; +using xrt::auxiliary::android::loadClassFromPackage; struct ipc_client_android { diff --git a/src/xrt/ipc/android/src/main/AndroidManifest.xml b/src/xrt/ipc/android/src/main/AndroidManifest.xml index b28f47840..4f8c19ed8 100644 --- a/src/xrt/ipc/android/src/main/AndroidManifest.xml +++ b/src/xrt/ipc/android/src/main/AndroidManifest.xml @@ -7,10 +7,12 @@ + android:foregroundServiceType="connectedDevice|mediaPlayback" + android:permission="org.khronos.openxr.permission.OPENXR"> + diff --git a/src/xrt/ipc/android/src/main/aidl/org/freedesktop/monado/ipc/IMonado.aidl b/src/xrt/ipc/android/src/main/aidl/org/freedesktop/monado/ipc/IMonado.aidl index 88bf3b33d..e06a56a4a 100644 --- a/src/xrt/ipc/android/src/main/aidl/org/freedesktop/monado/ipc/IMonado.aidl +++ b/src/xrt/ipc/android/src/main/aidl/org/freedesktop/monado/ipc/IMonado.aidl @@ -22,4 +22,14 @@ interface IMonado { * Provide the surface we inject into the activity, back to the service. */ void passAppSurface(in Surface surface); + + /*! + * Asking service to create surface and attach it to the display matches given display id. + */ + boolean createSurface(int displayId, boolean focusable); + + /*! + * Asking service whether it has the capbility to draw over other apps or not. + */ + boolean canDrawOverOtherApps(); } diff --git a/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/Client.java b/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/Client.java index f4f355bd2..d0bc2914f 100644 --- a/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/Client.java +++ b/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/Client.java @@ -20,12 +20,16 @@ import android.os.IBinder; import android.os.ParcelFileDescriptor; import android.os.RemoteException; import android.util.Log; +import android.view.Surface; import android.view.SurfaceHolder; +import android.view.WindowManager; import androidx.annotation.Keep; +import androidx.annotation.Nullable; import org.freedesktop.monado.auxiliary.MonadoView; import org.freedesktop.monado.auxiliary.NativeCounterpart; +import org.freedesktop.monado.auxiliary.SystemUiController; import java.io.IOException; @@ -38,9 +42,9 @@ import java.io.IOException; public class Client implements ServiceConnection { private static final String TAG = "monado-ipc-client"; /** - * Used to block native until we have our side of the socket pair. + * Used to block until binder is ready. */ - private final Object connectSync = new Object(); + private final Object binderSync = new Object(); /** * Keep track of the ipc_client_android instance over on the native side. */ @@ -76,8 +80,10 @@ public class Client implements ServiceConnection { * Intent for connecting to service */ private Intent intent = null; - - private SurfaceHolder surfaceHolder; + /** + * Controll system ui visibility + */ + private SystemUiController systemUiController = null; /** * Constructor @@ -100,8 +106,14 @@ public class Client implements ServiceConnection { } intent = null; - //! @todo do we close this first? - fd = null; + if (fd != null) { + try { + fd.close(); + } catch (IOException e) { + e.printStackTrace(); + } + fd = null; + } } /** @@ -118,6 +130,8 @@ public class Client implements ServiceConnection { *

* The IPC client code on Android should load this class (from the right package), instantiate * this class (retaining a reference to it!), and call this method. + *

+ * This method must not be called from the main (UI) thread. * * @param context_ Context to use to make the connection. (We get the application context * from it.) @@ -134,26 +148,75 @@ public class Client implements ServiceConnection { public int blockingConnect(Context context_, String packageName) { Log.i(TAG, "blockingConnect"); - Activity activity = (Activity) context_; - - MonadoView monadoView = MonadoView.attachToActivity(activity); - surfaceHolder = monadoView.waitGetSurfaceHolder(2000); - - synchronized (connectSync) { + synchronized (binderSync) { if (!bind(context_, packageName)) { Log.e(TAG, "Bind failed immediately"); // Bind failed immediately return -1; } try { - while (fd == null) { - connectSync.wait(); - } + binderSync.wait(); } catch (InterruptedException e) { Log.e(TAG, "Interrupted: " + e.toString()); return -1; } } + + if (monado == null) { + Log.e(TAG, "Invalid binder object"); + return -1; + } + + boolean surfaceCreated = false; + Activity activity = (Activity) context_; + + try { + // Determine whether runtime or client should create surface + if (monado.canDrawOverOtherApps()) { + WindowManager wm = (WindowManager) context_.getSystemService(Context.WINDOW_SERVICE); + surfaceCreated = monado.createSurface(wm.getDefaultDisplay().getDisplayId(), false); + } else { + Surface surface = attachViewAndGetSurface(activity); + surfaceCreated = (surface != null); + if (surfaceCreated) { + monado.passAppSurface(surface); + } + } + } catch (RemoteException e) { + e.printStackTrace(); + } + + if (!surfaceCreated) { + Log.e(TAG, "Failed to create surface"); + handleFailure(); + return -1; + } + + systemUiController = new SystemUiController(activity); + systemUiController.hide(); + + // Create socket pair + ParcelFileDescriptor theirs; + ParcelFileDescriptor ours; + try { + ParcelFileDescriptor[] fds = ParcelFileDescriptor.createSocketPair(); + ours = fds[0]; + theirs = fds[1]; + monado.connect(theirs); + } catch (IOException e) { + e.printStackTrace(); + Log.e(TAG, "could not create socket pair: " + e.toString()); + handleFailure(); + return -1; + } catch (RemoteException e) { + e.printStackTrace(); + Log.e(TAG, "could not connect to service: " + e.toString()); + handleFailure(); + return -1; + } + + fd = ours; + Log.i(TAG, "Socket fd " + fd.getFd()); return fd.getFd(); } @@ -215,12 +278,20 @@ public class Client implements ServiceConnection { shutdown(); } + @Nullable + private Surface attachViewAndGetSurface(Activity activity) { + MonadoView monadoView = MonadoView.attachToActivity(activity); + SurfaceHolder holder = monadoView.waitGetSurfaceHolder(2000); + Surface surface = null; + if (holder != null) { + surface = holder.getSurface(); + } + + return surface; + } + /** * Handle the asynchronous connection of the binder IPC. - *

- * This sets up the class member `monado`, as well as the member `fd`. It calls - * `IMonado.connect()` automatically. The client still needs to call `IMonado.passAppSurface()` - * on `monado`. * * @param name should match the intent above, but not used. * @param service the associated service, which we cast in this function. @@ -228,45 +299,10 @@ public class Client implements ServiceConnection { @Override public void onServiceConnected(ComponentName name, IBinder service) { Log.i(TAG, "onServiceConnected"); - monado = IMonado.Stub.asInterface(service); - try { - monado.passAppSurface(surfaceHolder.getSurface()); - } catch (RemoteException e) { - e.printStackTrace(); - Log.e(TAG, "Could not pass app surface: " + e.toString()); - } - - ParcelFileDescriptor theirs; - ParcelFileDescriptor ours; - try { - ParcelFileDescriptor[] fds = ParcelFileDescriptor.createSocketPair(); - ours = fds[0]; - theirs = fds[1]; - } catch (IOException e) { - e.printStackTrace(); - Log.e(TAG, "could not create socket pair: " + e.toString()); - handleFailure(); - return; - } - - Thread thread = new Thread(() -> { - try { - while(true) { - monado.connect(theirs); - } - } catch (RemoteException e) { - e.printStackTrace(); - Log.e(TAG, "could not call IMonado.connect: " + e.toString()); - handleFailure(); - } - }); - thread.start(); - - synchronized (connectSync) { - Log.e(TAG, String.format("Notifing connectSync with fd %d", ours.getFd())); - fd = ours; - connectSync.notify(); + synchronized (binderSync) { + monado = IMonado.Stub.asInterface(service); + binderSync.notify(); } } diff --git a/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/MonadoImpl.java b/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/MonadoImpl.java index 330661dbe..30d7f06f8 100644 --- a/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/MonadoImpl.java +++ b/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/MonadoImpl.java @@ -13,12 +13,15 @@ package org.freedesktop.monado.ipc; import android.os.ParcelFileDescriptor; import android.util.Log; import android.view.Surface; +import android.view.SurfaceHolder; import androidx.annotation.Keep; import androidx.annotation.NonNull; import org.jetbrains.annotations.NotNull; +import java.io.IOException; + /** * Java implementation of the IMonado IPC interface. *

@@ -39,10 +42,59 @@ public class MonadoImpl extends IMonado.Stub { System.loadLibrary("monado-service"); } + private final Thread compositorThread = new Thread( + this::threadEntry, + "CompositorThread"); + private boolean started = false; + + private SurfaceManager surfaceManager; + + public MonadoImpl(@NonNull SurfaceManager surfaceManager) { + this.surfaceManager = surfaceManager; + this.surfaceManager.setCallback(new SurfaceHolder.Callback() { + @Override + public void surfaceCreated(@NonNull SurfaceHolder holder) { + Log.i(TAG, "surfaceCreated"); + nativeAppSurface(holder.getSurface()); + } + + @Override + public void surfaceChanged(@NonNull SurfaceHolder holder, int format, int width, int height) { + } + + @Override + public void surfaceDestroyed(@NonNull SurfaceHolder holder) { + } + }); + } + + private void launchThreadIfNeeded() { + synchronized (compositorThread) { + if (!started) { + compositorThread.start(); + nativeWaitForServerStartup(); + started = true; + } + } + } + @Override public void connect(@NotNull ParcelFileDescriptor parcelFileDescriptor) { - Log.i(TAG, "connect"); - nativeAddClient(parcelFileDescriptor.detachFd()); + /// @todo launch this thread earlier/elsewhere + launchThreadIfNeeded(); + int fd = parcelFileDescriptor.getFd(); + Log.i(TAG, "connect: given fd " + fd); + if (nativeAddClient(fd) != 0) { + Log.e(TAG, "Failed to transfer client fd ownership!"); + try { + parcelFileDescriptor.close(); + } catch (IOException e) { + // do nothing, probably already closed. + } + } else { + Log.i(TAG, "connect: fd ownership transferred"); + parcelFileDescriptor.detachFd(); + } } @Override @@ -55,6 +107,36 @@ public class MonadoImpl extends IMonado.Stub { nativeAppSurface(surface); } + @Override + public boolean createSurface(int displayId, boolean focusable) { + Log.i(TAG, "createSurface"); + return surfaceManager.createSurfaceOnDisplay(displayId, focusable); + } + + @Override + public boolean canDrawOverOtherApps() { + Log.i(TAG, "canDrawOverOtherApps"); + return surfaceManager.canDrawOverlays(); + } + + private void threadEntry() { + Log.i(TAG, "threadEntry"); + nativeThreadEntry(); + Log.i(TAG, "native thread has exited"); + } + + /** + * Native thread entry point. + */ + @SuppressWarnings("JavaJniMissingFunction") + private native void nativeThreadEntry(); + + /** + * Native method that waits until the server reports that it is, in fact, started up. + */ + @SuppressWarnings("JavaJniMissingFunction") + private native void nativeWaitForServerStartup(); + /** * Native handling of receiving a surface: should convert it to an ANativeWindow then do stuff * with it. @@ -63,7 +145,7 @@ public class MonadoImpl extends IMonado.Stub { * See `src/xrt/targets/service-lib/service_target.cpp` for the implementation. * * @param surface The surface to pass to native code - * @todo figure out a good way to make the MonadoImpl pointer a client ID + * @todo figure out a good way to have a client ID */ @SuppressWarnings("JavaJniMissingFunction") private native void nativeAppSurface(@NonNull Surface surface); @@ -80,8 +162,9 @@ public class MonadoImpl extends IMonado.Stub { * See `src/xrt/targets/service-lib/service_target.cpp` for the implementation. * * @param fd The incoming file descriptor: ownership is transferred to native code here. - * @todo figure out a good way to make the MonadoImpl pointer a client ID + * @return 0 on success, anything else means the fd wasn't sent and ownership not transferred. + * @todo figure out a good way to have a client ID */ @SuppressWarnings("JavaJniMissingFunction") - private native void nativeAddClient(int fd); + private native int nativeAddClient(int fd); } diff --git a/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/MonadoService.kt b/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/MonadoService.kt index 7bebdedfd..306052d6a 100644 --- a/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/MonadoService.kt +++ b/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/MonadoService.kt @@ -26,11 +26,27 @@ import javax.inject.Inject */ @AndroidEntryPoint class MonadoService : Service() { - private val binder = MonadoImpl() + private val binder: MonadoImpl by lazy { + MonadoImpl(surfaceManager) + } @Inject lateinit var serviceNotification: IServiceNotification + private lateinit var surfaceManager: SurfaceManager + + override fun onCreate() { + super.onCreate() + + surfaceManager = SurfaceManager(this) + } + + override fun onDestroy() { + super.onDestroy() + + surfaceManager.destroySurface() + } + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { Log.d(TAG, "onStartCommand") // if this isn't a restart @@ -53,23 +69,27 @@ class MonadoService : Service() { } private fun handleStart() { - val pendingShutdownIntent = PendingIntent.getForegroundService(this, - 0, - Intent(BuildConfig.SHUTDOWN_ACTION).setPackage(packageName), - 0) + val pendingShutdownIntent = PendingIntent.getForegroundService( + this, + 0, + Intent(BuildConfig.SHUTDOWN_ACTION).setPackage(packageName), + 0 + ) val notification = serviceNotification.buildNotification(this, pendingShutdownIntent) - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - startForeground(serviceNotification.getNotificationId(), - notification, - ServiceInfo.FOREGROUND_SERVICE_TYPE_MANIFEST) + startForeground( + serviceNotification.getNotificationId(), + notification, + ServiceInfo.FOREGROUND_SERVICE_TYPE_MANIFEST + ) } else { - startForeground(serviceNotification.getNotificationId(), - notification) + startForeground( + serviceNotification.getNotificationId(), + notification + ) } - } companion object { diff --git a/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/SurfaceManager.kt b/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/SurfaceManager.kt new file mode 100644 index 000000000..3cf7c4113 --- /dev/null +++ b/src/xrt/ipc/android/src/main/java/org/freedesktop/monado/ipc/SurfaceManager.kt @@ -0,0 +1,219 @@ +// Copyright 2021, Qualcomm Innovation Center, Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Class that manages surface + * @author Jarvis Huang + * @ingroup ipc_android + */ +package org.freedesktop.monado.ipc + +import android.content.Context +import android.hardware.display.DisplayManager +import android.os.Handler +import android.os.Looper +import android.provider.Settings +import android.util.Log +import android.view.Display +import android.view.SurfaceHolder +import android.view.SurfaceView +import android.view.WindowManager +import androidx.annotation.UiThread +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.Condition +import java.util.concurrent.locks.ReentrantLock + +/** + * Class that creates/manages surface on display. + */ +class SurfaceManager(context: Context) : SurfaceHolder.Callback { + private val appContext: Context = context.applicationContext + private val surfaceLock: ReentrantLock = ReentrantLock() + private val surfaceCondition: Condition = surfaceLock.newCondition() + private var callback: SurfaceHolder.Callback? = null + private val uiHandler: Handler = Handler(Looper.getMainLooper()) + private val viewHelper: ViewHelper = ViewHelper(this) + + override fun surfaceCreated(holder: SurfaceHolder) { + Log.i(TAG, "surfaceCreated") + callback?.surfaceCreated(holder) + + surfaceLock.lock() + surfaceCondition.signal() + surfaceLock.unlock() + } + + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { + Log.i(TAG, "surfaceChanged, size: " + width + "x" + height) + callback?.surfaceChanged(holder, format, width, height) + } + + override fun surfaceDestroyed(holder: SurfaceHolder) { + Log.i(TAG, "surfaceDestroyed") + callback?.surfaceDestroyed(holder) + } + + /** + * Register a callback for surface status. + * + * @param callback Callback to be invoked. + */ + fun setCallback(callback: SurfaceHolder.Callback?) { + this.callback = callback + } + + /** + * Create surface on required display. + * + * @param displayId Target display id. + * @param focusable True if the surface should be focusable; otherwise false. + * @return True if operation succeeded. + */ + @Synchronized + fun createSurfaceOnDisplay(displayId: Int, focusable: Boolean): Boolean { + if (!canDrawOverlays()) { + Log.w(TAG, "Unable to draw over other apps!") + return false + } + + val dm = appContext.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager + val targetDisplay = dm.getDisplay(displayId) + if (targetDisplay == null) { + Log.w(TAG, "Can't find target display, id: $displayId") + return false + } + + if (viewHelper.hasSamePropertiesWithCurrentView(targetDisplay, focusable)) { + Log.i(TAG, "Reuse current surface") + return true + } + + if (Looper.getMainLooper().isCurrentThread) { + viewHelper.removeAndAddView(appContext, targetDisplay, focusable) + } else { + uiHandler.post { viewHelper.removeAndAddView(appContext, targetDisplay, focusable) } + surfaceLock.lock() + try { + surfaceCondition.await(1, TimeUnit.SECONDS) + } catch (exception: InterruptedException) { + exception.printStackTrace() + } finally { + Log.i(TAG, "surface ready") + surfaceLock.unlock() + } + } + return true + } + + /** + * Check if current process has the capability to draw over other applications. + * + * Implementation of [Settings.canDrawOverlays] checks both context and UID, + * therefore this cannot be done in client side. + * + * @return True if current process can draw over other applications; otherwise false. + */ + fun canDrawOverlays(): Boolean { + return Settings.canDrawOverlays(appContext) + } + + /** + * Destroy created surface. + */ + fun destroySurface() { + viewHelper.removeView() + } + + /** + * Helper class that manages surface view. + */ + private class ViewHelper(private val callback: SurfaceHolder.Callback) { + private var view: SurfaceView? = null + private var displayContext: Context? = null + + @UiThread + fun removeAndAddView(context: Context, targetDisplay: Display, focusable: Boolean) { + removeView() + addView(context, targetDisplay, focusable) + } + + @UiThread + fun addView(context: Context, display: Display, focusable: Boolean) { + // WindowManager is associated with display context. + Log.i(TAG, "Add view to display " + display.displayId) + displayContext = context.createDisplayContext(display) + addViewInternal(displayContext!!, focusable) + } + + @UiThread + fun removeView() { + if (view != null && displayContext != null) { + removeViewInternal(displayContext!!) + displayContext = null + } + } + + fun hasSamePropertiesWithCurrentView(display: Display, focusable: Boolean): Boolean { + return if (view == null || displayContext == null) { + false + } else { + isSameDisplay(displayContext!!, display) && !isFocusableChanged(focusable) + } + } + + /** + * Check whether given display is the one being used right now. + */ + @Suppress("DEPRECATION") + private fun isSameDisplay(context: Context, display: Display): Boolean { + val wm = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager + return wm.defaultDisplay != null && wm.defaultDisplay.displayId == display.displayId + } + + private fun isFocusableChanged(focusable: Boolean): Boolean { + val lp = view!!.layoutParams as WindowManager.LayoutParams + val currentFocusable = lp.flags == VIEW_FLAG_FOCUSABLE + return focusable != currentFocusable + } + + @UiThread + private fun addViewInternal(context: Context, focusable: Boolean) { + val v = SurfaceView(context) + v.holder.addCallback(callback) + val lp = WindowManager.LayoutParams() + lp.type = WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY + lp.flags = if (focusable) VIEW_FLAG_FOCUSABLE else VIEW_FLAG_NOT_FOCUSABLE + + val wm = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager + wm.addView(v, lp) + if (focusable) { + v.requestFocus() + } + + view = v + } + + @UiThread + private fun removeViewInternal(context: Context) { + val wm = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager + wm.removeView(view) + view = null + } + + companion object { + @Suppress("DEPRECATION") + private const val VIEW_FLAG_FOCUSABLE = WindowManager.LayoutParams.FLAG_FULLSCREEN + + @Suppress("DEPRECATION") + private const val VIEW_FLAG_NOT_FOCUSABLE = + WindowManager.LayoutParams.FLAG_FULLSCREEN or + WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE or + WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE + } + } + + companion object { + private const val TAG = "SurfaceManager" + } + +} diff --git a/src/xrt/ipc/client/ipc_client.h b/src/xrt/ipc/client/ipc_client.h index c73bc7d42..9e26af84a 100644 --- a/src/xrt/ipc/client/ipc_client.h +++ b/src/xrt/ipc/client/ipc_client.h @@ -74,7 +74,7 @@ struct ipc_connection * @param xina Optional native image allocator for client-side allocation. Takes * ownership if one is supplied. * @param xdev Taken in but not used currently @todo remove this param? - * @param[out] out_xcn Pointer to receive the created xrt_system_compositor. + * @param[out] out_xcs Pointer to receive the created xrt_system_compositor. */ int ipc_client_create_system_compositor(struct ipc_connection *ipc_c, diff --git a/src/xrt/ipc/client/ipc_client_compositor.c b/src/xrt/ipc/client/ipc_client_compositor.c index 6acf0e495..ec0f105f4 100644 --- a/src/xrt/ipc/client/ipc_client_compositor.c +++ b/src/xrt/ipc/client/ipc_client_compositor.c @@ -13,6 +13,7 @@ #include "xrt/xrt_defines.h" #include "util/u_misc.h" +#include "util/u_trace_marker.h" #include "os/os_time.h" @@ -64,8 +65,6 @@ struct ipc_client_compositor uint32_t slot_id; uint32_t num_layers; - - enum xrt_blend_mode env_blend_mode; } layers; //! Has the native compositor been created, only supports one for now. @@ -117,8 +116,8 @@ compositor_disconnect(struct ipc_connection *ipc_c) #define IPC_CALL_CHK(call) \ xrt_result_t res = (call); \ - if (res == XRT_ERROR_IPC_FAILURE) { \ - IPC_ERROR(icc->ipc_c, "Call error '%s'!", __func__); \ + if (res != XRT_SUCCESS) { \ + IPC_ERROR(icc->ipc_c, "Call error '%i'!", res); \ } static xrt_result_t @@ -208,12 +207,14 @@ swapchain_server_create(struct ipc_client_compositor *icc, uint32_t handle; uint32_t num_images; uint64_t size; + bool use_dedicated_allocation; r = ipc_call_swapchain_create(icc->ipc_c, // connection info, // in &handle, // out &num_images, // out &size, // out + &use_dedicated_allocation, // out remote_handles, // handles IPC_MAX_SWAPCHAIN_HANDLES); // handles if (r != XRT_SUCCESS) { @@ -226,12 +227,14 @@ swapchain_server_create(struct ipc_client_compositor *icc, ics->base.base.acquire_image = ipc_compositor_swapchain_acquire_image; ics->base.base.release_image = ipc_compositor_swapchain_release_image; ics->base.base.destroy = ipc_compositor_swapchain_destroy; + ics->base.base.reference.count = 1; ics->icc = icc; ics->id = handle; for (uint32_t i = 0; i < num_images; i++) { ics->base.images[i].handle = remote_handles[i]; ics->base.images[i].size = size; + ics->base.images[i].use_dedicated_allocation = use_dedicated_allocation; } *out_xsc = &ics->base.base; @@ -273,6 +276,7 @@ swapchain_server_import(struct ipc_client_compositor *icc, ics->base.base.acquire_image = ipc_compositor_swapchain_acquire_image; ics->base.base.release_image = ipc_compositor_swapchain_release_image; ics->base.base.destroy = ipc_compositor_swapchain_destroy; + ics->base.base.reference.count = 1; ics->icc = icc; ics->id = id; @@ -373,6 +377,8 @@ ipc_compositor_begin_session(struct xrt_compositor *xc, enum xrt_view_type view_ static xrt_result_t ipc_compositor_end_session(struct xrt_compositor *xc) { + IPC_TRACE_MARKER(); + struct ipc_client_compositor *icc = ipc_client_compositor(xc); IPC_TRACE(icc->ipc_c, "Compositor end session."); @@ -388,17 +394,16 @@ ipc_compositor_wait_frame(struct xrt_compositor *xc, uint64_t *out_predicted_display_time, uint64_t *out_predicted_display_period) { + IPC_TRACE_MARKER(); struct ipc_client_compositor *icc = ipc_client_compositor(xc); uint64_t wake_up_time_ns = 0; - uint64_t min_display_period_ns = 0; - IPC_CALL_CHK(ipc_call_compositor_wait_frame(icc->ipc_c, // Connection - out_frame_id, // Frame id - out_predicted_display_time, // Display time - &wake_up_time_ns, // When we should wake up - out_predicted_display_period, // Current period - &min_display_period_ns)); // Minimum display period + IPC_CALL_CHK(ipc_call_compositor_predict_frame(icc->ipc_c, // Connection + out_frame_id, // Frame id + &wake_up_time_ns, // When we should wake up + out_predicted_display_time, // Display time + out_predicted_display_period)); // Current period uint64_t now_ns = os_monotonic_get_ns(); @@ -452,11 +457,18 @@ ipc_compositor_begin_frame(struct xrt_compositor *xc, int64_t frame_id) } static xrt_result_t -ipc_compositor_layer_begin(struct xrt_compositor *xc, int64_t frame_id, enum xrt_blend_mode env_blend_mode) +ipc_compositor_layer_begin(struct xrt_compositor *xc, + int64_t frame_id, + uint64_t display_time_ns, + enum xrt_blend_mode env_blend_mode) { struct ipc_client_compositor *icc = ipc_client_compositor(xc); - icc->layers.env_blend_mode = env_blend_mode; + struct ipc_shared_memory *ism = icc->ipc_c->ism; + struct ipc_layer_slot *slot = &ism->slots[icc->layers.slot_id]; + + slot->display_time_ns = display_time_ns; + slot->env_blend_mode = env_blend_mode; return XRT_SUCCESS; } @@ -653,9 +665,39 @@ ipc_compositor_destroy(struct xrt_compositor *xc) assert(icc->compositor_created); + IPC_CALL_CHK(ipc_call_session_destroy(icc->ipc_c)); + icc->compositor_created = false; } +static void +ipc_compositor_init(struct ipc_client_compositor *icc, struct xrt_compositor_native **out_xcn) +{ + icc->base.base.create_swapchain = ipc_compositor_swapchain_create; + icc->base.base.import_swapchain = ipc_compositor_swapchain_import; + icc->base.base.begin_session = ipc_compositor_begin_session; + icc->base.base.end_session = ipc_compositor_end_session; + icc->base.base.wait_frame = ipc_compositor_wait_frame; + icc->base.base.begin_frame = ipc_compositor_begin_frame; + icc->base.base.discard_frame = ipc_compositor_discard_frame; + icc->base.base.layer_begin = ipc_compositor_layer_begin; + icc->base.base.layer_stereo_projection = ipc_compositor_layer_stereo_projection; + icc->base.base.layer_stereo_projection_depth = ipc_compositor_layer_stereo_projection_depth; + icc->base.base.layer_quad = ipc_compositor_layer_quad; + icc->base.base.layer_cube = ipc_compositor_layer_cube; + icc->base.base.layer_cylinder = ipc_compositor_layer_cylinder; + icc->base.base.layer_equirect1 = ipc_compositor_layer_equirect1; + icc->base.base.layer_equirect2 = ipc_compositor_layer_equirect2; + icc->base.base.layer_commit = ipc_compositor_layer_commit; + icc->base.base.destroy = ipc_compositor_destroy; + icc->base.base.poll_events = ipc_compositor_poll_events; + + // Fetch info from the compositor, among it the format format list. + get_info(&(icc->base.base), &icc->base.base.info); + + *out_xcn = &icc->base; +} + /* * @@ -782,11 +824,18 @@ ipc_syscomp_create_native_compositor(struct xrt_system_compositor *xsc, return XRT_ERROR_MULTI_SESSION_NOT_IMPLEMENTED; } - icc->compositor_created = true; - *out_xcn = &icc->base; - + // Needs to be done before init. IPC_CALL_CHK(ipc_call_session_create(icc->ipc_c, xsi)); + if (res != XRT_SUCCESS) { + return res; + } + + // Needs to be done after session create call. + ipc_compositor_init(icc, out_xcn); + + icc->compositor_created = true; + return XRT_SUCCESS; } @@ -819,24 +868,6 @@ ipc_client_create_system_compositor(struct ipc_connection *ipc_c, { struct ipc_client_compositor *c = U_TYPED_CALLOC(struct ipc_client_compositor); - c->base.base.create_swapchain = ipc_compositor_swapchain_create; - c->base.base.import_swapchain = ipc_compositor_swapchain_import; - c->base.base.begin_session = ipc_compositor_begin_session; - c->base.base.end_session = ipc_compositor_end_session; - c->base.base.wait_frame = ipc_compositor_wait_frame; - c->base.base.begin_frame = ipc_compositor_begin_frame; - c->base.base.discard_frame = ipc_compositor_discard_frame; - c->base.base.layer_begin = ipc_compositor_layer_begin; - c->base.base.layer_stereo_projection = ipc_compositor_layer_stereo_projection; - c->base.base.layer_stereo_projection_depth = ipc_compositor_layer_stereo_projection_depth; - c->base.base.layer_quad = ipc_compositor_layer_quad; - c->base.base.layer_cube = ipc_compositor_layer_cube; - c->base.base.layer_cylinder = ipc_compositor_layer_cylinder; - c->base.base.layer_equirect1 = ipc_compositor_layer_equirect1; - c->base.base.layer_equirect2 = ipc_compositor_layer_equirect2; - c->base.base.layer_commit = ipc_compositor_layer_commit; - c->base.base.destroy = ipc_compositor_destroy; - c->base.base.poll_events = ipc_compositor_poll_events; c->system.create_native_compositor = ipc_syscomp_create_native_compositor; c->system.destroy = ipc_syscomp_destroy; c->ipc_c = ipc_c; @@ -853,9 +884,6 @@ ipc_client_create_system_compositor(struct ipc_connection *ipc_c, } #endif - // Fetch info from the compositor, among it the format format list. - get_info(&(c->base.base), &c->base.base.info); - // Fetch info from the system compositor. get_system_info(c, &c->system.info); diff --git a/src/xrt/ipc/client/ipc_client_device.c b/src/xrt/ipc/client/ipc_client_device.c index a03587d1c..ae72ae226 100644 --- a/src/xrt/ipc/client/ipc_client_device.c +++ b/src/xrt/ipc/client/ipc_client_device.c @@ -107,12 +107,13 @@ void ipc_client_device_get_hand_tracking(struct xrt_device *xdev, enum xrt_input_name name, uint64_t at_timestamp_ns, - struct xrt_hand_joint_set *out_value) + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp_ns) { struct ipc_client_device *icd = ipc_client_device(xdev); - xrt_result_t r = - ipc_call_device_get_hand_tracking(icd->ipc_c, icd->device_id, name, at_timestamp_ns, out_value); + xrt_result_t r = ipc_call_device_get_hand_tracking(icd->ipc_c, icd->device_id, name, at_timestamp_ns, out_value, + out_timestamp_ns); if (r != XRT_SUCCESS) { IPC_ERROR(icd->ipc_c, "Error sending input update!"); } @@ -120,11 +121,12 @@ ipc_client_device_get_hand_tracking(struct xrt_device *xdev, static void ipc_client_device_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { // Empty + assert(false); } static void diff --git a/src/xrt/ipc/client/ipc_client_hmd.c b/src/xrt/ipc/client/ipc_client_hmd.c index a9ae77979..ef44e12c6 100644 --- a/src/xrt/ipc/client/ipc_client_hmd.c +++ b/src/xrt/ipc/client/ipc_client_hmd.c @@ -106,7 +106,7 @@ ipc_client_hmd_get_tracked_pose(struct xrt_device *xdev, static void ipc_client_hmd_get_view_pose(struct xrt_device *xdev, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { @@ -169,16 +169,17 @@ ipc_client_hmd_create(struct ipc_connection *ipc_c, struct xrt_tracking_origin * return NULL; } #endif + for (int i = 0; i < XRT_MAX_DEVICE_BLEND_MODES; i++) { + ich->base.hmd->blend_modes[i] = ipc_c->ism->hmd.blend_modes[i]; + } + ich->base.hmd->num_blend_modes = ipc_c->ism->hmd.num_blend_modes; - // clang-foramt off - ich->base.hmd->blend_mode = XRT_BLEND_MODE_OPAQUE; ich->base.hmd->views[0].display.w_pixels = ipc_c->ism->hmd.views[0].display.w_pixels; ich->base.hmd->views[0].display.h_pixels = ipc_c->ism->hmd.views[0].display.h_pixels; ich->base.hmd->views[0].fov = ipc_c->ism->hmd.views[0].fov; ich->base.hmd->views[1].display.w_pixels = ipc_c->ism->hmd.views[1].display.w_pixels; ich->base.hmd->views[1].display.h_pixels = ipc_c->ism->hmd.views[1].display.h_pixels; ich->base.hmd->views[1].fov = ipc_c->ism->hmd.views[1].fov; - // clang-foramt on // Distortion information, fills in xdev->compute_distortion(). u_distortion_mesh_set_none(&ich->base); diff --git a/src/xrt/ipc/client/ipc_client_instance.c b/src/xrt/ipc/client/ipc_client_instance.c index 94ff7cb5b..13fa14113 100644 --- a/src/xrt/ipc/client/ipc_client_instance.c +++ b/src/xrt/ipc/client/ipc_client_instance.c @@ -16,10 +16,12 @@ #include "util/u_misc.h" #include "util/u_var.h" #include "util/u_debug.h" +#include "util/u_git_tag.h" #include "shared/ipc_protocol.h" #include "client/ipc_client.h" #include "ipc_client_generated.h" +#include "util/u_file.h" #include #include @@ -28,9 +30,10 @@ #include #include #include +#include #include #include - +#include #ifdef XRT_GRAPHICS_BUFFER_HANDLE_IS_AHARDWAREBUFFER #include "android/android_ahardwarebuffer_allocator.h" @@ -42,6 +45,7 @@ #endif // XRT_OS_ANDROID DEBUG_GET_ONCE_LOG_OPTION(ipc_log, "IPC_LOG", U_LOGGING_WARN) +DEBUG_GET_ONCE_BOOL_OPTION(ipc_ignore_version, "IPC_IGNORE_VERSION", false) /* * @@ -91,6 +95,13 @@ ipc_connect(struct ipc_connection *ipc_c) IPC_ERROR(ipc_c, "Service Connect error!"); return false; } + // The ownership belongs to the Java object. Dup because the fd will be + // closed when client destroy. + socket = dup(socket); + if (socket < 0) { + IPC_ERROR(ipc_c, "Failed to dup fd with error %d!", errno); + return false; + } ipc_c->imc.socket_fd = socket; ipc_c->imc.ll = ipc_c->ll; @@ -118,13 +129,21 @@ ipc_connect(struct ipc_connection *ipc_c) int socket = ret; + char sock_file[PATH_MAX]; + + int size = u_file_get_path_in_runtime_dir(IPC_MSG_SOCK_FILE, sock_file, PATH_MAX); + if (size == -1) { + IPC_ERROR(ipc_c, "Could not get socket file name"); + return -1; + } + memset(&addr, 0, sizeof(addr)); addr.sun_family = AF_UNIX; - strcpy(addr.sun_path, IPC_MSG_SOCK_FILE); + strcpy(addr.sun_path, sock_file); ret = connect(socket, (struct sockaddr *)&addr, sizeof(addr)); if (ret < 0) { - IPC_ERROR(ipc_c, "Socket Connect error!"); + IPC_ERROR(ipc_c, "Failed to connec to socket %s: %s!", sock_file, strerror(errno)); close(socket); return false; } @@ -288,6 +307,16 @@ ipc_instance_create(struct xrt_instance_info *i_info, struct xrt_instance **out_ return -1; } + if (strncmp(u_git_tag, ii->ipc_c.ism->u_git_tag, IPC_VERSION_NAME_LEN) != 0) { + IPC_ERROR((&ii->ipc_c), "Monado client library version %s does not match service version %s", u_git_tag, + ii->ipc_c.ism->u_git_tag); + if (!debug_get_bool_option_ipc_ignore_version()) { + IPC_ERROR((&ii->ipc_c), "Set IPC_IGNORE_VERSION=1 to ignore this version conflict"); + free(ii); + return -1; + } + } + uint32_t count = 0; struct xrt_tracking_origin *xtrack = NULL; struct ipc_shared_memory *ism = ii->ipc_c.ism; diff --git a/src/xrt/ipc/meson.build b/src/xrt/ipc/meson.build index 1c5223537..aeeb2e5e5 100644 --- a/src/xrt/ipc/meson.build +++ b/src/xrt/ipc/meson.build @@ -64,6 +64,7 @@ lib_ipc_server = static_library( 'server/ipc_server_handler.c', 'server/ipc_server_per_client_thread.c', 'server/ipc_server_process.c', + 'server/ipc_server_mainloop_linux.c', ], include_directories: [ xrt_include, diff --git a/src/xrt/ipc/server/ipc_server.h b/src/xrt/ipc/server/ipc_server.h index 2ef91cf67..f5296533a 100644 --- a/src/xrt/ipc/server/ipc_server.h +++ b/src/xrt/ipc/server/ipc_server.h @@ -1,10 +1,11 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Common server side code. * @author Pete Black * @author Jakob Bornecrantz + * @author Ryan Pavlik * @ingroup ipc_server */ @@ -12,7 +13,6 @@ #include "xrt/xrt_compiler.h" -#include "util/u_render_timing.h" #include "util/u_logging.h" #include "os/os_threading.h" @@ -68,14 +68,6 @@ struct ipc_swapchain_data bool active; }; - -struct ipc_queued_event -{ - bool pending; - uint64_t timestamp; - union xrt_compositor_event event; -}; - /*! * Holds the state for a single client. * @@ -104,17 +96,7 @@ struct ipc_client_state //! Socket fd used for client comms struct ipc_message_channel imc; - //! State for rendering. - struct ipc_layer_slot render_state; - - //! Whether we are currently rendering @ref render_state - bool rendering_state; - - //! The frame timing state. - struct u_rt_helper urth; - struct ipc_app_state client_state; - struct ipc_queued_event queued_events[IPC_EVENT_QUEUE_SIZE]; int server_thread_index; }; @@ -147,6 +129,136 @@ struct ipc_device bool io_active; }; +/*! + * Platform-specific mainloop object for the IPC server. + * + * Contents are essentially implementation details, but are listed in full here so they may be included by value in the + * main ipc_server struct. + * + * @see ipc_design + * + * @ingroup ipc_server + */ +struct ipc_server_mainloop +{ + +#if defined(XRT_OS_ANDROID) || defined(XRT_OS_LINUX) || defined(XRT_DOXYGEN) + //! For waiting on various events in the main thread. + int epoll_fd; +#endif + +#if defined(XRT_OS_ANDROID) || defined(XRT_DOXYGEN) + /*! + * @name Android Mainloop Members + * @{ + */ + + //! File descriptor for the read end of our pipe for submitting new clients + int pipe_read; + + /*! + * File descriptor for the write end of our pipe for submitting new clients + * + * Must hold client_push_mutex while writing. + */ + int pipe_write; + + /*! + * Mutex for being able to register oneself as a new client. + * + * Locked only by threads in `ipc_server_mainloop_add_fd()`. + * + * This must be locked first, and kept locked the entire time a client is attempting to register and wait for + * confirmation. It ensures no acknowledgements of acceptance are lost and moves the overhead of ensuring this + * to the client thread. + */ + pthread_mutex_t client_push_mutex; + + + /*! + * The last client fd we accepted, to acknowledge client acceptance. + * + * Also used as a sentinel during shutdown. + * + * Must hold accept_mutex while writing. + */ + int last_accepted_fd; + + /*! + * Condition variable for accepting clients. + * + * Signalled when @ref last_accepted_fd is updated. + * + * Associated with @ref accept_mutex + */ + pthread_cond_t accept_cond; + + /*! + * Mutex for accepting clients. + * + * Locked by both clients and server: that is, by threads in `ipc_server_mainloop_add_fd()` and in the + * server/compositor thread in an implementation function called from `ipc_server_mainloop_poll()`. + * + * Exists to operate in conjunction with @ref accept_cond - it exists to make sure that the client can be woken + * when the server accepts it. + */ + pthread_mutex_t accept_mutex; + + + /*! @} */ +#define XRT_IPC_GOT_IMPL +#endif + +#if (defined(XRT_OS_LINUX) && !defined(XRT_OS_ANDROID)) || defined(XRT_DOXYGEN) + /*! + * @name Desktop Linux Mainloop Members + * @{ + */ + + //! Socket that we accept connections on. + int listen_socket; + + //! Were we launched by socket activation, instead of explicitly? + bool launched_by_socket; + + //! The socket filename we bound to, if any. + char *socket_filename; + + /*! @} */ + +#define XRT_IPC_GOT_IMPL +#endif + +#ifndef XRT_IPC_GOT_IMPL +#error "Need port" +#endif +}; + +/*! + * De-initialize the mainloop object. + * @public @memberof ipc_server_mainloop + */ +void +ipc_server_mainloop_deinit(struct ipc_server_mainloop *ml); + +/*! + * Initialize the mainloop object. + * + * @return <0 on error. + * @public @memberof ipc_server_mainloop + */ +int +ipc_server_mainloop_init(struct ipc_server_mainloop *ml); + +/*! + * @brief Poll the mainloop. + * + * Any errors are signalled by calling ipc_server_handle_failure() + * @public @memberof ipc_server_mainloop + */ +void +ipc_server_mainloop_poll(struct ipc_server *vs, struct ipc_server_mainloop *ml); + /*! * Main IPC object for the server. * @@ -156,6 +268,9 @@ struct ipc_server { struct xrt_instance *xinst; + //! Handle for the current process, e.g. pidfile on linux + struct u_process *process; + /* ---- HACK ---- */ void *hack; /* ---- HACK ---- */ @@ -163,8 +278,6 @@ struct ipc_server //! System compositor. struct xrt_system_compositor *xsysc; - //! Native compositor. - struct xrt_compositor_native *xcn; struct ipc_device idevs[IPC_SERVER_NUM_XDEVS]; struct xrt_tracking_origin *xtracks[IPC_SERVER_NUM_XDEVS]; @@ -172,11 +285,7 @@ struct ipc_server struct ipc_shared_memory *ism; xrt_shmem_handle_t ism_handle; - //! Socket that we accept connections on. - int listen_socket; - - //! For waiting on various events in the main thread. - int epoll_fd; + struct ipc_server_mainloop ml; // Is the mainloop supposed to run. volatile bool running; @@ -184,23 +293,23 @@ struct ipc_server // Should we exit when a client disconnects. bool exit_on_disconnect; - //! Were we launched by socket activation, instead of explicitly? - bool launched_by_socket; - - //! The socket filename we bound to, if any. - char *socket_filename; - enum u_logging_level ll; struct ipc_thread threads[IPC_MAX_CLIENTS]; volatile uint32_t current_slot_index; - int active_client_index; - int last_active_client_index; - struct os_mutex global_state_lock; + struct + { + int active_client_index; + int last_active_client_index; + + struct os_mutex lock; + } global_state; }; + +#ifndef XRT_OS_ANDROID /*! * Main entrypoint to the compositor process. * @@ -208,24 +317,54 @@ struct ipc_server */ int ipc_server_main(int argc, char **argv); +#endif +#ifdef XRT_OS_ANDROID /*! - * Android entry point to the IPC server process. + * Main entrypoint to the server process. + * + * @param ps Pointer to populate with the server struct. + * @param startup_complete_callback Function to call upon completing startup and populating *ps, but before entering the + * mainloop. + * @param data user data to pass to your callback. * * @ingroup ipc_server */ -#ifdef XRT_OS_ANDROID int -ipc_server_main_android(int fd); +ipc_server_main_android(struct ipc_server **ps, void (*startup_complete_callback)(void *data), void *data); #endif /*! - * Called by client threads to manage global state + * Set the new active client. * * @ingroup ipc_server */ void -update_server_state(struct ipc_server *vs); +ipc_server_set_active_client(struct ipc_server *s, int active_client_index); + +/*! + * Called by client threads to set a session to active. + * + * @ingroup ipc_server + */ +void +ipc_server_activate_session(volatile struct ipc_client_state *ics); + +/*! + * Called by client threads to set a session to deactivate. + * + * @ingroup ipc_server + */ +void +ipc_server_deactivate_session(volatile struct ipc_client_state *ics); + +/*! + * Called by client threads to recalculate active client. + * + * @ingroup ipc_server + */ +void +ipc_server_update_state(struct ipc_server *s); /*! * Thread function for the client side dispatching. @@ -235,6 +374,42 @@ update_server_state(struct ipc_server *vs); void * ipc_server_client_thread(void *_cs); +/*! + * This destroyes the native compositor for this client and any extra objects + * created from it, like all of the swapchains. + */ +void +ipc_server_client_destroy_compositor(volatile struct ipc_client_state *ics); + +/*! + * @defgroup ipc_server_internals Server Internals + * @brief These are only called by the platform-specific mainloop polling code. + * @ingroup ipc_server + * @{ + */ +/*! + * Start a thread for a client connected at the other end of the file descriptor @p fd. + * @memberof ipc_server + */ +void +ipc_server_start_client_listener_thread(struct ipc_server *vs, int fd); + +/*! + * Perform whatever needs to be done when the mainloop polling encounters a failure. + * @memberof ipc_server + */ +void +ipc_server_handle_failure(struct ipc_server *vs); + +/*! + * Perform whatever needs to be done when the mainloop polling identifies that the server should be shut down. + * + * Does something like setting a flag or otherwise signalling for shutdown: does not itself explicitly exit. + * @memberof ipc_server + */ +void +ipc_server_handle_shutdown_signal(struct ipc_server *vs); +//! @} /* * diff --git a/src/xrt/ipc/server/ipc_server_handler.c b/src/xrt/ipc/server/ipc_server_handler.c index 068462249..b8dec1cac 100644 --- a/src/xrt/ipc/server/ipc_server_handler.c +++ b/src/xrt/ipc/server/ipc_server_handler.c @@ -1,4 +1,4 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -10,6 +10,8 @@ #include "xrt/xrt_gfx_native.h" #include "util/u_misc.h" +#include "util/u_handles.h" +#include "util/u_trace_marker.h" #include "server/ipc_server.h" #include "ipc_server_generated.h" @@ -73,10 +75,13 @@ ipc_handle_instance_get_shm_fd(volatile struct ipc_client_state *ics, xrt_shmem_handle_t *out_handles, uint32_t *out_num_handles) { + IPC_TRACE_MARKER(); + assert(max_num_handles >= 1); out_handles[0] = ics->server->ism_handle; *out_num_handles = 1; + return XRT_SUCCESS; } @@ -84,6 +89,8 @@ xrt_result_t ipc_handle_system_compositor_get_info(volatile struct ipc_client_state *ics, struct xrt_system_compositor_info *out_info) { + IPC_TRACE_MARKER(); + *out_info = ics->server->xsysc->info; return XRT_SUCCESS; @@ -92,98 +99,404 @@ ipc_handle_system_compositor_get_info(volatile struct ipc_client_state *ics, xrt_result_t ipc_handle_session_create(volatile struct ipc_client_state *ics, const struct xrt_session_info *xsi) { - ics->client_state.session_active = false; - ics->client_state.session_overlay = false; - ics->client_state.session_visible = false; + IPC_TRACE_MARKER(); - if (xsi->is_overlay) { - ics->client_state.session_overlay = true; - ics->client_state.z_order = xsi->z_order; + struct xrt_compositor_native *xcn = NULL; + + if (ics->xc != NULL) { + return XRT_ERROR_IPC_SESSION_ALREADY_CREATED; } - update_server_state(ics->server); + xrt_result_t xret = xrt_syscomp_create_native_compositor(ics->server->xsysc, xsi, &xcn); + if (xret != XRT_SUCCESS) { + return xret; + } + + ics->client_state.session_overlay = xsi->is_overlay; + ics->client_state.z_order = xsi->z_order; + + ics->xc = &xcn->base; + + xrt_syscomp_set_state(ics->server->xsysc, ics->xc, ics->client_state.session_visible, + ics->client_state.session_focused); + xrt_syscomp_set_z_order(ics->server->xsysc, ics->xc, ics->client_state.z_order); + return XRT_SUCCESS; } xrt_result_t ipc_handle_session_begin(volatile struct ipc_client_state *ics) { - // ics->client_state.session_active = true; - // update_server_state(ics->server); - return XRT_SUCCESS; + IPC_TRACE_MARKER(); + + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } + + return xrt_comp_begin_session(ics->xc, 0); } xrt_result_t ipc_handle_session_end(volatile struct ipc_client_state *ics) { - ics->client_state.session_active = false; - update_server_state(ics->server); + IPC_TRACE_MARKER(); + + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } + + return xrt_comp_end_session(ics->xc); +} + +xrt_result_t +ipc_handle_session_destroy(volatile struct ipc_client_state *ics) +{ + IPC_TRACE_MARKER(); + + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } + + ipc_server_client_destroy_compositor(ics); + return XRT_SUCCESS; } xrt_result_t ipc_handle_compositor_get_info(volatile struct ipc_client_state *ics, struct xrt_compositor_info *out_info) { + IPC_TRACE_MARKER(); + + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } + *out_info = ics->xc->info; return XRT_SUCCESS; } xrt_result_t -ipc_handle_compositor_wait_frame(volatile struct ipc_client_state *ics, - int64_t *out_frame_id, - uint64_t *predicted_display_time, - uint64_t *wake_up_time, - uint64_t *predicted_display_period, - uint64_t *min_display_period) +ipc_handle_compositor_predict_frame(volatile struct ipc_client_state *ics, + int64_t *out_frame_id, + uint64_t *out_wake_up_time_ns, + uint64_t *out_predicted_display_time_ns, + uint64_t *out_predicted_display_period_ns) { - os_mutex_lock(&ics->server->global_state_lock); + IPC_TRACE_MARKER(); - u_rt_helper_predict((struct u_rt_helper *)&ics->urth, out_frame_id, predicted_display_time, wake_up_time, - predicted_display_period, min_display_period); + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } - os_mutex_unlock(&ics->server->global_state_lock); + /* + * We use this to signal that the session has started, this is needed + * to make this client/session active/visible/focused. + */ + ipc_server_activate_session(ics); - ics->client_state.session_active = true; - update_server_state(ics->server); - - return XRT_SUCCESS; + uint64_t gpu_time_ns = 0; + return xrt_comp_predict_frame( // + ics->xc, // + out_frame_id, // + out_wake_up_time_ns, // + &gpu_time_ns, // + out_predicted_display_time_ns, // + out_predicted_display_period_ns); // } xrt_result_t ipc_handle_compositor_wait_woke(volatile struct ipc_client_state *ics, int64_t frame_id) { - os_mutex_lock(&ics->server->global_state_lock); + IPC_TRACE_MARKER(); - u_rt_helper_mark_wait_woke((struct u_rt_helper *)&ics->urth, frame_id); + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } - os_mutex_unlock(&ics->server->global_state_lock); - - return XRT_SUCCESS; + return xrt_comp_mark_frame(ics->xc, frame_id, XRT_COMPOSITOR_FRAME_POINT_WOKE, os_monotonic_get_ns()); } xrt_result_t ipc_handle_compositor_begin_frame(volatile struct ipc_client_state *ics, int64_t frame_id) { - os_mutex_lock(&ics->server->global_state_lock); + IPC_TRACE_MARKER(); - u_rt_helper_mark_begin((struct u_rt_helper *)&ics->urth, frame_id); + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } - os_mutex_unlock(&ics->server->global_state_lock); - - return XRT_SUCCESS; + return xrt_comp_begin_frame(ics->xc, frame_id); } xrt_result_t ipc_handle_compositor_discard_frame(volatile struct ipc_client_state *ics, int64_t frame_id) { - os_mutex_lock(&ics->server->global_state_lock); + IPC_TRACE_MARKER(); - u_rt_helper_mark_discarded((struct u_rt_helper *)&ics->urth, frame_id); + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } - os_mutex_unlock(&ics->server->global_state_lock); + return xrt_comp_discard_frame(ics->xc, frame_id); +} - return XRT_SUCCESS; +static bool +_update_projection_layer(struct xrt_compositor *xc, + volatile struct ipc_client_state *ics, + volatile struct ipc_layer_entry *layer, + uint32_t i) +{ + // xdev + uint32_t device_id = layer->xdev_id; + // left + uint32_t lxsci = layer->swapchain_ids[0]; + // right + uint32_t rxsci = layer->swapchain_ids[1]; + + struct xrt_device *xdev = get_xdev(ics, device_id); + struct xrt_swapchain *lxcs = ics->xscs[lxsci]; + struct xrt_swapchain *rxcs = ics->xscs[rxsci]; + + if (lxcs == NULL || rxcs == NULL) { + U_LOG_E("Invalid swap chain for projection layer!"); + return false; + } + + if (xdev == NULL) { + U_LOG_E("Invalid xdev for projection layer!"); + return false; + } + + // Cast away volatile. + struct xrt_layer_data *data = (struct xrt_layer_data *)&layer->data; + + xrt_comp_layer_stereo_projection(xc, xdev, lxcs, rxcs, data); + + return true; +} + +static bool +_update_projection_layer_depth(struct xrt_compositor *xc, + volatile struct ipc_client_state *ics, + volatile struct ipc_layer_entry *layer, + uint32_t i) +{ + // xdev + uint32_t xdevi = layer->xdev_id; + // left + uint32_t l_xsci = layer->swapchain_ids[0]; + // right + uint32_t r_xsci = layer->swapchain_ids[1]; + // left + uint32_t l_d_xsci = layer->swapchain_ids[2]; + // right + uint32_t r_d_xsci = layer->swapchain_ids[3]; + + struct xrt_device *xdev = get_xdev(ics, xdevi); + struct xrt_swapchain *l_xcs = ics->xscs[l_xsci]; + struct xrt_swapchain *r_xcs = ics->xscs[r_xsci]; + struct xrt_swapchain *l_d_xcs = ics->xscs[l_d_xsci]; + struct xrt_swapchain *r_d_xcs = ics->xscs[r_d_xsci]; + + if (l_xcs == NULL || r_xcs == NULL || l_d_xcs == NULL || r_d_xcs == NULL) { + U_LOG_E("Invalid swap chain for projection layer #%u!", i); + return false; + } + + if (xdev == NULL) { + U_LOG_E("Invalid xdev for projection layer #%u!", i); + return false; + } + + // Cast away volatile. + struct xrt_layer_data *data = (struct xrt_layer_data *)&layer->data; + + xrt_comp_layer_stereo_projection_depth(xc, xdev, l_xcs, r_xcs, l_d_xcs, r_d_xcs, data); + + return true; +} + +static bool +do_single(struct xrt_compositor *xc, + volatile struct ipc_client_state *ics, + volatile struct ipc_layer_entry *layer, + uint32_t i, + const char *name, + struct xrt_device **out_xdev, + struct xrt_swapchain **out_xcs, + struct xrt_layer_data **out_data) +{ + uint32_t device_id = layer->xdev_id; + uint32_t sci = layer->swapchain_ids[0]; + + struct xrt_device *xdev = get_xdev(ics, device_id); + struct xrt_swapchain *xcs = ics->xscs[sci]; + + if (xcs == NULL) { + U_LOG_E("Invalid swapchain for layer #%u, '%s'!", i, name); + return false; + } + + if (xdev == NULL) { + U_LOG_E("Invalid xdev for layer #%u, '%s'!", i, name); + return false; + } + + // Cast away volatile. + struct xrt_layer_data *data = (struct xrt_layer_data *)&layer->data; + + *out_xdev = xdev; + *out_xcs = xcs; + *out_data = data; + + return true; +} + +static bool +_update_quad_layer(struct xrt_compositor *xc, + volatile struct ipc_client_state *ics, + volatile struct ipc_layer_entry *layer, + uint32_t i) +{ + struct xrt_device *xdev; + struct xrt_swapchain *xcs; + struct xrt_layer_data *data; + + if (!do_single(xc, ics, layer, i, "quad", &xdev, &xcs, &data)) { + return false; + } + + xrt_comp_layer_quad(xc, xdev, xcs, data); + + return true; +} + +static bool +_update_cube_layer(struct xrt_compositor *xc, + volatile struct ipc_client_state *ics, + volatile struct ipc_layer_entry *layer, + uint32_t i) +{ + struct xrt_device *xdev; + struct xrt_swapchain *xcs; + struct xrt_layer_data *data; + + if (!do_single(xc, ics, layer, i, "cube", &xdev, &xcs, &data)) { + return false; + } + + xrt_comp_layer_cube(xc, xdev, xcs, data); + + return true; +} + +static bool +_update_cylinder_layer(struct xrt_compositor *xc, + volatile struct ipc_client_state *ics, + volatile struct ipc_layer_entry *layer, + uint32_t i) +{ + struct xrt_device *xdev; + struct xrt_swapchain *xcs; + struct xrt_layer_data *data; + + if (!do_single(xc, ics, layer, i, "cylinder", &xdev, &xcs, &data)) { + return false; + } + + xrt_comp_layer_cylinder(xc, xdev, xcs, data); + + return true; +} + +static bool +_update_equirect1_layer(struct xrt_compositor *xc, + volatile struct ipc_client_state *ics, + volatile struct ipc_layer_entry *layer, + uint32_t i) +{ + struct xrt_device *xdev; + struct xrt_swapchain *xcs; + struct xrt_layer_data *data; + + if (!do_single(xc, ics, layer, i, "equirect1", &xdev, &xcs, &data)) { + return false; + } + + xrt_comp_layer_equirect1(xc, xdev, xcs, data); + + return true; +} + +static bool +_update_equirect2_layer(struct xrt_compositor *xc, + volatile struct ipc_client_state *ics, + volatile struct ipc_layer_entry *layer, + uint32_t i) +{ + struct xrt_device *xdev; + struct xrt_swapchain *xcs; + struct xrt_layer_data *data; + + if (!do_single(xc, ics, layer, i, "equirect2", &xdev, &xcs, &data)) { + return false; + } + + xrt_comp_layer_equirect2(xc, xdev, xcs, data); + + return true; +} + +static bool +_update_layers(volatile struct ipc_client_state *ics, struct xrt_compositor *xc, struct ipc_layer_slot *slot) +{ + IPC_TRACE_MARKER(); + + for (uint32_t i = 0; i < slot->num_layers; i++) { + volatile struct ipc_layer_entry *layer = &slot->layers[i]; + + switch (layer->data.type) { + case XRT_LAYER_STEREO_PROJECTION: + if (!_update_projection_layer(xc, ics, layer, i)) { + return false; + } + break; + case XRT_LAYER_STEREO_PROJECTION_DEPTH: + if (!_update_projection_layer_depth(xc, ics, layer, i)) { + return false; + } + break; + case XRT_LAYER_QUAD: + if (!_update_quad_layer(xc, ics, layer, i)) { + return false; + } + break; + case XRT_LAYER_CUBE: + if (!_update_cube_layer(xc, ics, layer, i)) { + return false; + } + break; + case XRT_LAYER_CYLINDER: + if (!_update_cylinder_layer(xc, ics, layer, i)) { + return false; + } + break; + case XRT_LAYER_EQUIRECT1: + if (!_update_equirect1_layer(xc, ics, layer, i)) { + return false; + } + break; + case XRT_LAYER_EQUIRECT2: + if (!_update_equirect2_layer(xc, ics, layer, i)) { + return false; + } + break; + default: U_LOG_E("Unhandled layer type '%i'!", layer->data.type); break; + } + } + + return true; } xrt_result_t @@ -194,33 +507,53 @@ ipc_handle_compositor_layer_sync(volatile struct ipc_client_state *ics, const xrt_graphics_sync_handle_t *handles, const uint32_t num_handles) { - struct ipc_shared_memory *ism = ics->server->ism; - struct ipc_layer_slot *slot = &ism->slots[slot_id]; + IPC_TRACE_MARKER(); - for (uint32_t i = 0; i < num_handles; i++) { - if (!xrt_graphics_sync_handle_is_valid(handles[i])) { - continue; - } -#ifdef XRT_GRAPHICS_SYNC_HANDLE_IS_FD - close(handles[i]); -#else -#error "Need port to transport these graphics buffers" -#endif + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; } - // Copy current slot data to our state. - ics->render_state = *slot; - ics->rendering_state = true; + struct ipc_shared_memory *ism = ics->server->ism; + struct ipc_layer_slot *slot = &ism->slots[slot_id]; + xrt_graphics_sync_handle_t sync_handle = XRT_GRAPHICS_SYNC_HANDLE_INVALID; - os_mutex_lock(&ics->server->global_state_lock); + // If we have one or more save the first handle. + if (num_handles >= 1) { + sync_handle = handles[0]; + } + + // Free all sync handles after the first one. + for (uint32_t i = 1; i < num_handles; i++) { + // Checks for valid handle. + xrt_graphics_sync_handle_t tmp = handles[i]; + u_graphics_sync_unref(&tmp); + } + + // Copy current slot data. + struct ipc_layer_slot copy = *slot; + + + /* + * Transfer data to underlying compositor. + */ + + xrt_comp_layer_begin(ics->xc, frame_id, copy.display_time_ns, copy.env_blend_mode); + + _update_layers(ics, ics->xc, ©); + + xrt_comp_layer_commit(ics->xc, frame_id, sync_handle); + + + /* + * Manage shared state. + */ + + os_mutex_lock(&ics->server->global_state.lock); *out_free_slot_id = (ics->server->current_slot_index + 1) % IPC_MAX_SLOTS; ics->server->current_slot_index = *out_free_slot_id; - // Also protected by the global lock. - u_rt_helper_mark_delivered((struct u_rt_helper *)&ics->urth, frame_id); - - os_mutex_unlock(&ics->server->global_state_lock); + os_mutex_unlock(&ics->server->global_state.lock); return XRT_SUCCESS; } @@ -228,25 +561,13 @@ ipc_handle_compositor_layer_sync(volatile struct ipc_client_state *ics, xrt_result_t ipc_handle_compositor_poll_events(volatile struct ipc_client_state *ics, union xrt_compositor_event *out_xce) { - uint64_t l_timestamp = UINT64_MAX; - volatile struct ipc_queued_event *event_to_send = NULL; - for (uint32_t i = 0; i < IPC_EVENT_QUEUE_SIZE; i++) { - volatile struct ipc_queued_event *e = &ics->queued_events[i]; - if (e->pending == true && e->timestamp < l_timestamp) { - event_to_send = e; - } + IPC_TRACE_MARKER(); + + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; } - // We always return an event in response to this call - - // We signal no events with a special event type. - out_xce->type = XRT_COMPOSITOR_EVENT_NONE; - - if (event_to_send) { - *out_xce = event_to_send->event; - event_to_send->pending = false; - } - - return XRT_SUCCESS; + return xrt_comp_poll_events(ics->xc, out_xce); } xrt_result_t @@ -268,7 +589,7 @@ ipc_handle_system_get_client_info(volatile struct ipc_client_state *_ics, //@todo: track this data in the ipc_client_state struct out_client_desc->primary_application = false; - if (ics->server->active_client_index == (int)id) { + if (ics->server->global_state.active_client_index == (int)id) { out_client_desc->primary_application = true; } @@ -280,6 +601,14 @@ ipc_handle_system_set_client_info(volatile struct ipc_client_state *ics, struct { ics->client_state.info = client_desc->info; ics->client_state.pid = client_desc->pid; + + IPC_INFO(ics->server, + "Client info\n" + "\tapplication_name: '%s'\n" + "\tpid: %i", + client_desc->info.application_name, // + client_desc->pid); // + return XRT_SUCCESS; } @@ -295,10 +624,10 @@ ipc_handle_system_get_clients(volatile struct ipc_client_state *_ics, struct ipc xrt_result_t ipc_handle_system_set_primary_client(volatile struct ipc_client_state *ics, uint32_t client_id) { - - ics->server->active_client_index = client_id; IPC_INFO(ics->server, "System setting active client to %d.", client_id); - update_server_state(ics->server); + + ipc_server_set_active_client(ics->server, client_id); + return XRT_SUCCESS; } @@ -350,10 +679,13 @@ ipc_handle_swapchain_create(volatile struct ipc_client_state *ics, uint32_t *out_id, uint32_t *out_num_images, uint64_t *out_size, + bool *out_use_dedicated_allocation, uint32_t max_num_handles, xrt_graphics_buffer_handle_t *out_handles, uint32_t *out_num_handles) { + IPC_TRACE_MARKER(); + xrt_result_t xret = XRT_SUCCESS; uint32_t index = 0; @@ -363,7 +695,7 @@ ipc_handle_swapchain_create(volatile struct ipc_client_state *ics, } // Create the swapchain - struct xrt_swapchain *xsc = NULL; + struct xrt_swapchain *xsc = NULL; // Has to be NULL. xret = xrt_comp_create_swapchain(ics->xc, info, &xsc); if (xret != XRT_SUCCESS) { IPC_ERROR(ics->server, "Error xrt_comp_create_swapchain failed!"); @@ -384,8 +716,16 @@ ipc_handle_swapchain_create(volatile struct ipc_client_state *ics, assert(xsc->num_images <= IPC_MAX_SWAPCHAIN_HANDLES); assert(xsc->num_images <= max_num_handles); - *out_id = index; + // Paranoia. + for (size_t i = 1; i < xsc->num_images; i++) { + assert(xscn->images[0].size == xscn->images[i].size); + assert(xscn->images[0].use_dedicated_allocation == xscn->images[i].use_dedicated_allocation); + } + + // Assuming all images allocated in the same swapchain have the same allocation requirements. *out_size = xscn->images[0].size; + *out_use_dedicated_allocation = xscn->images[0].use_dedicated_allocation; + *out_id = index; *out_num_images = xsc->num_images; // Setup the fds. @@ -405,6 +745,8 @@ ipc_handle_swapchain_import(volatile struct ipc_client_state *ics, const xrt_graphics_buffer_handle_t *handles, uint32_t num_handles) { + IPC_TRACE_MARKER(); + xrt_result_t xret = XRT_SUCCESS; uint32_t index = 0; @@ -440,6 +782,10 @@ ipc_handle_swapchain_import(volatile struct ipc_client_state *ics, xrt_result_t ipc_handle_swapchain_wait_image(volatile struct ipc_client_state *ics, uint32_t id, uint64_t timeout, uint32_t index) { + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } + //! @todo Look up the index. uint32_t sc_index = id; struct xrt_swapchain *xsc = ics->xscs[sc_index]; @@ -452,6 +798,10 @@ ipc_handle_swapchain_wait_image(volatile struct ipc_client_state *ics, uint32_t xrt_result_t ipc_handle_swapchain_acquire_image(volatile struct ipc_client_state *ics, uint32_t id, uint32_t *out_index) { + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } + //! @todo Look up the index. uint32_t sc_index = id; struct xrt_swapchain *xsc = ics->xscs[sc_index]; @@ -464,6 +814,10 @@ ipc_handle_swapchain_acquire_image(volatile struct ipc_client_state *ics, uint32 xrt_result_t ipc_handle_swapchain_release_image(volatile struct ipc_client_state *ics, uint32_t id, uint32_t index) { + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } + //! @todo Look up the index. uint32_t sc_index = id; struct xrt_swapchain *xsc = ics->xscs[sc_index]; @@ -476,10 +830,14 @@ ipc_handle_swapchain_release_image(volatile struct ipc_client_state *ics, uint32 xrt_result_t ipc_handle_swapchain_destroy(volatile struct ipc_client_state *ics, uint32_t id) { - //! @todo Implement destroy swapchain. + if (ics->xc == NULL) { + return XRT_ERROR_IPC_SESSION_NOT_CREATED; + } + ics->num_swapchains--; - xrt_swapchain_destroy((struct xrt_swapchain **)&ics->xscs[id]); + // Drop our reference, does NULL checking. Cast away volatile. + xrt_swapchain_reference((struct xrt_swapchain **)&ics->xscs[id], NULL); ics->swapchain_data[id].active = false; return XRT_SUCCESS; @@ -546,7 +904,6 @@ ipc_handle_device_get_tracked_pose(volatile struct ipc_client_state *ics, uint64_t at_timestamp, struct xrt_space_relation *out_relation) { - // To make the code a bit more readable. uint32_t device_id = id; struct ipc_device *isdev = &ics->server->idevs[device_id]; @@ -583,7 +940,8 @@ ipc_handle_device_get_hand_tracking(volatile struct ipc_client_state *ics, uint32_t id, enum xrt_input_name name, uint64_t at_timestamp, - struct xrt_hand_joint_set *out_value) + struct xrt_hand_joint_set *out_value, + uint64_t *out_timestamp) { // To make the code a bit more readable. @@ -591,14 +949,15 @@ ipc_handle_device_get_hand_tracking(volatile struct ipc_client_state *ics, struct xrt_device *xdev = get_xdev(ics, device_id); // Get the pose. - xrt_device_get_hand_tracking(xdev, name, at_timestamp, out_value); + xrt_device_get_hand_tracking(xdev, name, at_timestamp, out_value, out_timestamp); return XRT_SUCCESS; } + xrt_result_t ipc_handle_device_get_view_pose(volatile struct ipc_client_state *ics, uint32_t id, - struct xrt_vec3 *eye_relation, + const struct xrt_vec3 *eye_relation, uint32_t view_index, struct xrt_pose *out_pose) { diff --git a/src/xrt/ipc/server/ipc_server_mainloop_android.c b/src/xrt/ipc/server/ipc_server_mainloop_android.c new file mode 100644 index 000000000..d87080198 --- /dev/null +++ b/src/xrt/ipc/server/ipc_server_mainloop_android.c @@ -0,0 +1,218 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Server mainloop details on Android. + * @author Ryan Pavlik + * @author Pete Black + * @author Jakob Bornecrantz + * @ingroup ipc_server + */ + +#include "xrt/xrt_config_have.h" +#include "xrt/xrt_config_os.h" + +#include "os/os_time.h" +#include "util/u_var.h" +#include "util/u_misc.h" +#include "util/u_debug.h" + +#include "server/ipc_server.h" +#include "server/ipc_server_mainloop_android.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SHUTTING_DOWN (-1) + +/* + * + * Static functions. + * + */ + +static int +init_pipe(struct ipc_server_mainloop *ml) +{ + int pipefd[2]; + int ret = pipe(pipefd); + if (ret < 0) { + U_LOG_E("pipe2() failed '%i'", ret); + return ret; + } + ml->pipe_read = pipefd[0]; + ml->pipe_write = pipefd[1]; + return 0; +} + +static int +init_epoll(struct ipc_server_mainloop *ml) +{ + int ret = epoll_create1(EPOLL_CLOEXEC); + if (ret < 0) { + return ret; + } + + pthread_mutex_init(&ml->client_push_mutex, NULL); + pthread_cond_init(&ml->accept_cond, NULL); + pthread_mutex_init(&ml->accept_mutex, NULL); + ml->epoll_fd = ret; + + struct epoll_event ev = {0}; + + + ev.events = EPOLLIN; + ev.data.fd = ml->pipe_read; + ret = epoll_ctl(ml->epoll_fd, EPOLL_CTL_ADD, ml->pipe_read, &ev); + if (ret < 0) { + U_LOG_E("epoll_ctl(pipe_read) failed '%i'", ret); + return ret; + } + + return 0; +} + + +static void +handle_listen(struct ipc_server *vs, struct ipc_server_mainloop *ml) +{ + int newfd = 0; + pthread_mutex_lock(&ml->accept_mutex); + if (read(ml->pipe_read, &newfd, sizeof(newfd)) == sizeof(newfd)) { + // client_push_mutex should prevent dropping acknowledgements + assert(ml->last_accepted_fd == 0); + // Release the thread that gave us this fd. + ml->last_accepted_fd = newfd; + ipc_server_start_client_listener_thread(vs, newfd); + pthread_cond_broadcast(&ml->accept_cond); + } else { + U_LOG_E("error on pipe read"); + ipc_server_handle_failure(vs); + return; + } + pthread_mutex_unlock(&ml->accept_mutex); +} + +#define NUM_POLL_EVENTS 8 +#define NO_SLEEP 0 + +/* + * + * Exported functions + * + */ +void +ipc_server_mainloop_poll(struct ipc_server *vs, struct ipc_server_mainloop *ml) +{ + int epoll_fd = ml->epoll_fd; + + struct epoll_event events[NUM_POLL_EVENTS] = {0}; + + // No sleeping, returns immediately. + int ret = epoll_wait(epoll_fd, events, NUM_POLL_EVENTS, NO_SLEEP); + if (ret < 0) { + U_LOG_E("epoll_wait failed with '%i'.", ret); + ipc_server_handle_failure(vs); + return; + } + + for (int i = 0; i < ret; i++) { + // Somebody new at the door. + if (events[i].data.fd == ml->pipe_read) { + handle_listen(vs, ml); + } + } +} + +int +ipc_server_mainloop_init(struct ipc_server_mainloop *ml) +{ + int ret = init_pipe(ml); + if (ret < 0) { + ipc_server_mainloop_deinit(ml); + return ret; + } + + ret = init_epoll(ml); + if (ret < 0) { + ipc_server_mainloop_deinit(ml); + return ret; + } + + return 0; +} + +void +ipc_server_mainloop_deinit(struct ipc_server_mainloop *ml) +{ + if (ml == NULL) { + return; + } + if (ml->pipe_read > 0) { + // Close pipe on exit + close(ml->pipe_read); + ml->pipe_read = -1; + } + //! @todo close pipe_write or epoll_fd? + + // Tell everybody we're done and they should go away. + pthread_mutex_lock(&ml->accept_mutex); + while (ml->last_accepted_fd != 0) { + // Don't accidentally intervene in somebody else's message, + // wait until there's no unblocks pending. + pthread_cond_wait(&ml->accept_cond, &ml->accept_mutex); + } + ml->last_accepted_fd = SHUTTING_DOWN; + pthread_cond_broadcast(&ml->accept_cond); + pthread_mutex_unlock(&ml->accept_mutex); +} + +int +ipc_server_mainloop_add_fd(struct ipc_server *vs, struct ipc_server_mainloop *ml, int newfd) +{ + // Take the client push lock here, serializing clients attempting to connect. + // This one won't be unlocked when waiting on the condition variable, ensuring we keep other clients out. + pthread_mutex_lock(&ml->client_push_mutex); + + // Take the lock here, so we don't accidentally miss our fd being accepted. + pthread_mutex_lock(&ml->accept_mutex); + + // Write our fd number: the other side of the pipe is in the same process, so passing just the number is OK. + int ret = write(ml->pipe_write, &newfd, sizeof(newfd)); + if (ret < 0) { + U_LOG_E("write to pipe failed with '%i'.", ret); + goto exit; + } + + // Normal looping on the condition variable's condition. + while (ml->last_accepted_fd != newfd && ml->last_accepted_fd != SHUTTING_DOWN) { + ret = pthread_cond_wait(&ml->accept_cond, &ml->accept_mutex); + if (ret < 0) { + U_LOG_E("pthread_cond_wait failed with '%i'.", ret); + goto exit; + } + } + if (ml->last_accepted_fd == SHUTTING_DOWN) { + // we actually didn't hand off our client, we should error out. + U_LOG_W("server was shutting down."); + ret = -1; + } else { + // OK, we have now been accepted. Zero out the last accepted fd. + ml->last_accepted_fd = 0; + ret = 0; + } +exit: + pthread_mutex_unlock(&ml->accept_mutex); + pthread_mutex_unlock(&ml->client_push_mutex); + return ret; +} diff --git a/src/xrt/ipc/server/ipc_server_mainloop_android.h b/src/xrt/ipc/server/ipc_server_mainloop_android.h new file mode 100644 index 000000000..c920d89ad --- /dev/null +++ b/src/xrt/ipc/server/ipc_server_mainloop_android.h @@ -0,0 +1,30 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Additional server entry points needed for Android. + * @author Ryan Pavlik + * @ingroup ipc_server + */ + +#pragma once + +#include "ipc_server.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * Pass an fd for a new client to the mainloop. + * + * @see ipc_design + * @public @memberof ipc_server_mainloop + */ +int +ipc_server_mainloop_add_fd(struct ipc_server *vs, struct ipc_server_mainloop *ml, int newfd); + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/ipc/server/ipc_server_mainloop_linux.c b/src/xrt/ipc/server/ipc_server_mainloop_linux.c new file mode 100644 index 000000000..376e09dda --- /dev/null +++ b/src/xrt/ipc/server/ipc_server_mainloop_linux.c @@ -0,0 +1,294 @@ +// Copyright 2020-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Server mainloop details on Linux. + * @author Pete Black + * @author Jakob Bornecrantz + * @author Ryan Pavlik + * @ingroup ipc_server + */ + +#include "xrt/xrt_device.h" +#include "xrt/xrt_instance.h" +#include "xrt/xrt_compositor.h" +#include "xrt/xrt_config_have.h" +#include "xrt/xrt_config_os.h" + +#include "os/os_time.h" +#include "util/u_var.h" +#include "util/u_misc.h" +#include "util/u_debug.h" +#include "util/u_trace_marker.h" +#include "util/u_file.h" + +#include "shared/ipc_shmem.h" +#include "server/ipc_server.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef XRT_HAVE_SYSTEMD +#include +#endif + + +/* + * + * Static functions. + * + */ +static int +get_systemd_socket(struct ipc_server_mainloop *ml, int *out_fd) +{ +#ifdef XRT_HAVE_SYSTEMD + // We may have been launched with socket activation + int num_fds = sd_listen_fds(0); + if (num_fds > 1) { + U_LOG_E("Too many file descriptors passed by systemd."); + return -1; + } + if (num_fds == 1) { + *out_fd = SD_LISTEN_FDS_START + 0; + ml->launched_by_socket = true; + U_LOG_D("Got existing socket from systemd."); + } +#endif + return 0; +} + +static int +create_listen_socket(struct ipc_server_mainloop *ml, int *out_fd) +{ + // no fd provided + struct sockaddr_un addr; + int fd, ret; + + fd = socket(PF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + U_LOG_E("Message Socket Create Error!"); + return fd; + } + + + char sock_file[PATH_MAX]; + + int size = u_file_get_path_in_runtime_dir(IPC_MSG_SOCK_FILE, sock_file, PATH_MAX); + if (size == -1) { + U_LOG_E("Could not get socket file name"); + return -1; + } + + memset(&addr, 0, sizeof(addr)); + + addr.sun_family = AF_UNIX; + strcpy(addr.sun_path, sock_file); + + ret = bind(fd, (struct sockaddr *)&addr, sizeof(addr)); + +#ifdef XRT_HAVE_LIBBSD + // no other instance is running, or we would have never arrived here + if (ret < 0 && errno == EADDRINUSE) { + U_LOG_W("Removing stale socket file %s", sock_file); + + ret = unlink(sock_file); + if (ret < 0) { + U_LOG_E("Failed to remove stale socket file %s: %s", sock_file, strerror(errno)); + return ret; + } + ret = bind(fd, (struct sockaddr *)&addr, sizeof(addr)); + } +#endif + + if (ret < 0) { + U_LOG_E("Could not bind socket to path %s: %s. Is the service running already?", sock_file, + strerror(errno)); +#ifdef XRT_HAVE_SYSTEMD + U_LOG_E("Or, is the systemd unit monado.socket or monado-dev.socket active?"); +#endif + if (errno == EADDRINUSE) { + U_LOG_E("If monado-service is not running, delete %s before starting a new instance", + sock_file); + } + close(fd); + return ret; + } + // Save for later + ml->socket_filename = strdup(sock_file); + + ret = listen(fd, IPC_MAX_CLIENTS); + if (ret < 0) { + close(fd); + return ret; + } + U_LOG_D("Created listening socket %s.", sock_file); + *out_fd = fd; + return 0; +} + +static int +init_listen_socket(struct ipc_server_mainloop *ml) +{ + int fd = -1, ret; + ml->listen_socket = -1; + + ret = get_systemd_socket(ml, &fd); + if (ret < 0) { + return ret; + } + + if (fd == -1) { + ret = create_listen_socket(ml, &fd); + if (ret < 0) { + return ret; + } + } + // All ok! + ml->listen_socket = fd; + U_LOG_D("Listening socket is fd %d", ml->listen_socket); + + return fd; +} + +static int +init_epoll(struct ipc_server_mainloop *ml) +{ + int ret = epoll_create1(EPOLL_CLOEXEC); + if (ret < 0) { + return ret; + } + + ml->epoll_fd = ret; + + struct epoll_event ev = {0}; + + if (!ml->launched_by_socket) { + // Can't do this when launched by systemd socket activation by + // default. + // This polls stdin. + ev.events = EPOLLIN; + ev.data.fd = 0; // stdin + ret = epoll_ctl(ml->epoll_fd, EPOLL_CTL_ADD, 0, &ev); + if (ret < 0) { + U_LOG_E("epoll_ctl(stdin) failed '%i'", ret); + return ret; + } + } + + ev.events = EPOLLIN; + ev.data.fd = ml->listen_socket; + ret = epoll_ctl(ml->epoll_fd, EPOLL_CTL_ADD, ml->listen_socket, &ev); + if (ret < 0) { + U_LOG_E("epoll_ctl(listen_socket) failed '%i'", ret); + return ret; + } + + return 0; +} + +static void +handle_listen(struct ipc_server *vs, struct ipc_server_mainloop *ml) +{ + int ret = accept(ml->listen_socket, NULL, NULL); + if (ret < 0) { + U_LOG_E("accept '%i'", ret); + ipc_server_handle_failure(vs); + return; + } + ipc_server_start_client_listener_thread(vs, ret); +} + +#define NUM_POLL_EVENTS 8 +#define NO_SLEEP 0 + +/* + * + * Exported functions + * + */ + +void +ipc_server_mainloop_poll(struct ipc_server *vs, struct ipc_server_mainloop *ml) +{ + IPC_TRACE_MARKER(); + + int epoll_fd = ml->epoll_fd; + + struct epoll_event events[NUM_POLL_EVENTS] = {0}; + + // No sleeping, returns immediately. + int ret = epoll_wait(epoll_fd, events, NUM_POLL_EVENTS, NO_SLEEP); + if (ret < 0) { + U_LOG_E("epoll_wait failed with '%i'.", ret); + ipc_server_handle_failure(vs); + return; + } + + for (int i = 0; i < ret; i++) { + // If we get data on stdin, stop. + if (events[i].data.fd == 0) { + ipc_server_handle_shutdown_signal(vs); + return; + } + + // Somebody new at the door. + if (events[i].data.fd == ml->listen_socket) { + handle_listen(vs, ml); + } + } +} + +int +ipc_server_mainloop_init(struct ipc_server_mainloop *ml) +{ + IPC_TRACE_MARKER(); + + int ret = init_listen_socket(ml); + if (ret < 0) { + ipc_server_mainloop_deinit(ml); + return ret; + } + + ret = init_epoll(ml); + if (ret < 0) { + ipc_server_mainloop_deinit(ml); + return ret; + } + return 0; +} + +void +ipc_server_mainloop_deinit(struct ipc_server_mainloop *ml) +{ + IPC_TRACE_MARKER(); + + if (ml == NULL) { + return; + } + if (ml->listen_socket > 0) { + // Close socket on exit + close(ml->listen_socket); + ml->listen_socket = -1; + if (!ml->launched_by_socket && ml->socket_filename) { + // Unlink it too, but only if we bound it. + unlink(ml->socket_filename); + free(ml->socket_filename); + ml->socket_filename = NULL; + } + } + //! @todo close epoll_fd? +} diff --git a/src/xrt/ipc/server/ipc_server_per_client_thread.c b/src/xrt/ipc/server/ipc_server_per_client_thread.c index 1ae776783..adafec590 100644 --- a/src/xrt/ipc/server/ipc_server_per_client_thread.c +++ b/src/xrt/ipc/server/ipc_server_per_client_thread.c @@ -1,4 +1,4 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -69,9 +69,6 @@ client_loop(volatile struct ipc_client_state *ics) { IPC_INFO(ics->server, "Client connected"); - // Make sure it's ready for the client. - u_rt_helper_client_clear((struct u_rt_helper *)&ics->urth); - // Claim the client fd. int epoll_fd = setup_epoll(ics); if (epoll_fd < 0) { @@ -123,65 +120,55 @@ client_loop(volatile struct ipc_client_state *ics) epoll_fd = -1; // Multiple threads might be looking at these fields. - os_mutex_lock(&ics->server->global_state_lock); + os_mutex_lock(&ics->server->global_state.lock); ipc_message_channel_close((struct ipc_message_channel *)&ics->imc); - // Reset the urth for the next client. - u_rt_helper_client_clear((struct u_rt_helper *)&ics->urth); - - ics->num_swapchains = 0; - ics->server->threads[ics->server_thread_index].state = IPC_THREAD_STOPPING; ics->server_thread_index = -1; memset((void *)&ics->client_state, 0, sizeof(struct ipc_app_state)); - // Make sure to reset the renderstate fully. - ics->rendering_state = false; - ics->render_state.num_layers = 0; - for (uint32_t i = 0; i < ARRAY_SIZE(ics->render_state.layers); ++i) { - volatile struct ipc_layer_entry *rl = &ics->render_state.layers[i]; + os_mutex_unlock(&ics->server->global_state.lock); - rl->swapchain_ids[0] = 0; - rl->swapchain_ids[1] = 0; - rl->data.flip_y = false; - /*! - * @todo this is redundant, we're setting both elements of a - * union. Why? Can we just zero the whole render_state? - */ - rl->data.stereo.l.sub.image_index = 0; - rl->data.stereo.r.sub.image_index = 0; - rl->data.quad.sub.image_index = 0; - rl->data.cube.sub.image_index = 0; - rl->data.cylinder.sub.image_index = 0; - rl->data.equirect1.sub.image_index = 0; - rl->data.equirect2.sub.image_index = 0; - - //! @todo set rects or array index? - } - - // Destroy all swapchains now. - for (uint32_t j = 0; j < IPC_MAX_CLIENT_SWAPCHAINS; j++) { - xrt_swapchain_destroy((struct xrt_swapchain **)&ics->xscs[j]); - ics->swapchain_data[j].active = false; - IPC_TRACE(ics->server, "Destroyed swapchain %d.", j); - } - - os_mutex_unlock(&ics->server->global_state_lock); + ipc_server_client_destroy_compositor(ics); // Should we stop the server when a client disconnects? if (ics->server->exit_on_disconnect) { ics->server->running = false; } + + ipc_server_deactivate_session(ics); } /* * - * Entry point. + * 'Exported' functions. * */ +void +ipc_server_client_destroy_compositor(volatile struct ipc_client_state *ics) +{ + // Multiple threads might be looking at these fields. + os_mutex_lock(&ics->server->global_state.lock); + + ics->num_swapchains = 0; + + // Destroy all swapchains now. + for (uint32_t j = 0; j < IPC_MAX_CLIENT_SWAPCHAINS; j++) { + // Drop our reference, does NULL checking. Cast away volatile. + xrt_swapchain_reference((struct xrt_swapchain **)&ics->xscs[j], NULL); + ics->swapchain_data[j].active = false; + IPC_TRACE(ics->server, "Destroyed swapchain %d.", j); + } + + os_mutex_unlock(&ics->server->global_state.lock); + + // Cast away volatile. + xrt_comp_destroy((struct xrt_compositor **)&ics->xc); +} + void * ipc_server_client_thread(void *_ics) { @@ -189,7 +176,5 @@ ipc_server_client_thread(void *_ics) client_loop(ics); - update_server_state(ics->server); - return NULL; } diff --git a/src/xrt/ipc/server/ipc_server_process.c b/src/xrt/ipc/server/ipc_server_process.c index 02bf580d1..344d27325 100644 --- a/src/xrt/ipc/server/ipc_server_process.c +++ b/src/xrt/ipc/server/ipc_server_process.c @@ -1,10 +1,11 @@ -// Copyright 2020, Collabora, Ltd. +// Copyright 2020-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Server process functions. * @author Pete Black * @author Jakob Bornecrantz + * @author Ryan Pavlik * @ingroup ipc_server */ @@ -18,6 +19,11 @@ #include "util/u_var.h" #include "util/u_misc.h" #include "util/u_debug.h" +#include "util/u_trace_marker.h" +#include "util/u_verify.h" +#include "util/u_process.h" + +#include "util/u_git_tag.h" #include "shared/ipc_shmem.h" #include "server/ipc_server.h" @@ -31,28 +37,24 @@ #include #include #include -#include #include #include #include #include #include -#ifdef XRT_HAVE_SYSTEMD -#include -#endif - /* ---- HACK ---- */ extern int oxr_sdl2_hack_create(void **out_hack); extern void -oxr_sdl2_hack_start(void *hack, struct xrt_instance *xinst); +oxr_sdl2_hack_start(void *hack, struct xrt_instance *xinst, struct xrt_device **xdevs); extern void oxr_sdl2_hack_stop(void **hack_ptr); /* ---- HACK ---- */ + /* * * Defines and helpers. @@ -62,11 +64,12 @@ oxr_sdl2_hack_stop(void **hack_ptr); DEBUG_GET_ONCE_BOOL_OPTION(exit_on_disconnect, "IPC_EXIT_ON_DISCONNECT", false) DEBUG_GET_ONCE_LOG_OPTION(ipc_log, "IPC_LOG", U_LOGGING_WARN) -struct _z_sort_data -{ - int32_t index; - int32_t z_order; -}; + +/* + * + * Idev functions. + * + */ static void init_idev(struct ipc_device *idev, struct xrt_device *xdev) @@ -98,8 +101,6 @@ teardown_all(struct ipc_server *s) { u_var_remove_root(s); - xrt_comp_native_destroy(&s->xcn); - xrt_syscomp_destroy(&s->xsysc); for (size_t i = 0; i < IPC_SERVER_NUM_XDEVS; i++) { @@ -108,19 +109,10 @@ teardown_all(struct ipc_server *s) xrt_instance_destroy(&s->xinst); - if (s->listen_socket > 0) { - // Close socket on exit - close(s->listen_socket); - s->listen_socket = -1; - if (!s->launched_by_socket && s->socket_filename) { - // Unlink it too, but only if we bound it. - unlink(s->socket_filename); - free(s->socket_filename); - s->socket_filename = NULL; - } - } + ipc_server_mainloop_deinit(&s->ml); - os_mutex_destroy(&s->global_state_lock); + os_mutex_destroy(&s->global_state.lock); + u_process_destroy(s->process); } static int @@ -264,6 +256,13 @@ init_shm(struct ipc_server *s) ism->hmd.views[1].display.w_pixels = xdev->hmd->views[1].display.w_pixels; ism->hmd.views[1].display.h_pixels = xdev->hmd->views[1].display.h_pixels; ism->hmd.views[1].fov = xdev->hmd->views[1].fov; + + for (size_t i = 0; i < xdev->hmd->num_blend_modes; i++) { + // Not super necessary, we also do this assert in oxr_system.c + assert(u_verify_blend_mode_valid(xdev->hmd->blend_modes[i])); + ism->hmd.blend_modes[i] = xdev->hmd->blend_modes[i]; + } + ism->hmd.num_blend_modes = xdev->hmd->num_blend_modes; } // Setup the tracking origin. @@ -323,139 +322,31 @@ init_shm(struct ipc_server *s) // Finally tell the client how many devices we have. s->ism->num_isdevs = count; - return 0; -} - -static int -get_systemd_socket(struct ipc_server *s, int *out_fd) -{ -#ifdef XRT_HAVE_SYSTEMD - // We may have been launched with socket activation - int num_fds = sd_listen_fds(0); - if (num_fds > 1) { - U_LOG_E("Too many file descriptors passed by systemd."); - return -1; - } - if (num_fds == 1) { - *out_fd = SD_LISTEN_FDS_START + 0; - s->launched_by_socket = true; - U_LOG_D("Got existing socket from systemd."); - } -#endif - return 0; -} - -static int -create_listen_socket(struct ipc_server *s, int *out_fd) -{ - // no fd provided - struct sockaddr_un addr; - int fd, ret; - - fd = socket(PF_UNIX, SOCK_STREAM, 0); - if (fd < 0) { - U_LOG_E("Message Socket Create Error!"); - return fd; - } - - memset(&addr, 0, sizeof(addr)); - - addr.sun_family = AF_UNIX; - strcpy(addr.sun_path, IPC_MSG_SOCK_FILE); - - ret = bind(fd, (struct sockaddr *)&addr, sizeof(addr)); - if (ret < 0) { - U_LOG_E( - "Could not bind socket to path %s: is the " - "service running already?", - IPC_MSG_SOCK_FILE); -#ifdef XRT_HAVE_SYSTEMD - U_LOG_E( - "Or, is the systemd unit monado.socket or " - "monado-dev.socket active?"); -#endif - close(fd); - return ret; - } - // Save for later - s->socket_filename = strdup(IPC_MSG_SOCK_FILE); - - ret = listen(fd, IPC_MAX_CLIENTS); - if (ret < 0) { - close(fd); - return ret; - } - U_LOG_D("Created listening socket."); - *out_fd = fd; - return 0; -} - -static int -init_listen_socket(struct ipc_server *s) -{ - int fd = -1, ret; - s->listen_socket = -1; - - ret = get_systemd_socket(s, &fd); - if (ret < 0) { - return ret; - } - - if (fd == -1) { - ret = create_listen_socket(s, &fd); - if (ret < 0) { - return ret; - } - } - // All ok! - s->listen_socket = fd; - U_LOG_D("Listening socket is fd '%d'.", s->listen_socket); - - return fd; -} - -static int -init_epoll(struct ipc_server *s) -{ - int ret = epoll_create1(EPOLL_CLOEXEC); - if (ret < 0) { - return ret; - } - - s->epoll_fd = ret; - - struct epoll_event ev = {0}; - - if (!s->launched_by_socket) { - // Can't do this when launched by systemd socket activation by - // default - ev.events = EPOLLIN; - ev.data.fd = 0; // stdin - ret = epoll_ctl(s->epoll_fd, EPOLL_CTL_ADD, 0, &ev); - if (ret < 0) { - U_LOG_E("Error epoll_ctl(stdin) failed '%i'!", ret); - return ret; - } - } - - ev.events = EPOLLIN; - ev.data.fd = s->listen_socket; - ret = epoll_ctl(s->epoll_fd, EPOLL_CTL_ADD, s->listen_socket, &ev); - if (ret < 0) { - U_LOG_E("Error epoll_ctl(listen_socket) failed '%i'!", ret); - return ret; - } + snprintf(s->ism->u_git_tag, IPC_VERSION_NAME_LEN, "%s", u_git_tag); return 0; } -static void -start_client_listener_thread(struct ipc_server *vs, int fd) +void +ipc_server_handle_failure(struct ipc_server *vs) +{ + // Right now handled just the same as a graceful shutdown. + vs->running = false; +} + +void +ipc_server_handle_shutdown_signal(struct ipc_server *vs) +{ + vs->running = false; +} + +void +ipc_server_start_client_listener_thread(struct ipc_server *vs, int fd) { volatile struct ipc_client_state *ics = NULL; int32_t cs_index = -1; - os_mutex_lock(&vs->global_state_lock); + os_mutex_lock(&vs->global_state.lock); // find the next free thread in our array (server_thread_index is -1) // and have it handle this connection @@ -471,7 +362,7 @@ start_client_listener_thread(struct ipc_server *vs, int fd) close(fd); // Unlock when we are done. - os_mutex_unlock(&vs->global_state_lock); + os_mutex_unlock(&vs->global_state.lock); U_LOG_E("Max client count reached!"); return; @@ -483,7 +374,7 @@ start_client_listener_thread(struct ipc_server *vs, int fd) close(fd); // Unlock when we are done. - os_mutex_unlock(&vs->global_state_lock); + os_mutex_unlock(&vs->global_state.lock); U_LOG_E("Client state management error!"); return; @@ -503,12 +394,20 @@ start_client_listener_thread(struct ipc_server *vs, int fd) os_thread_start(&it->thread, ipc_server_client_thread, (void *)ics); // Unlock when we are done. - os_mutex_unlock(&vs->global_state_lock); + os_mutex_unlock(&vs->global_state.lock); } static int init_all(struct ipc_server *s) { + s->process = u_process_create_if_not_running(); + + if (!s->process) { + U_LOG_E("monado-service is already running! Use XRT_LOG=trace for more information."); + teardown_all(s); + return 1; + } + // Yes we should be running. s->running = true; s->exit_on_disconnect = debug_get_bool_option_exit_on_disconnect(); @@ -554,43 +453,19 @@ init_all(struct ipc_server *s) return ret; } - struct xrt_session_info xsi = { - .is_overlay = false, - .flags = 0, - .z_order = 0, - }; - ret = xrt_syscomp_create_native_compositor(s->xsysc, &xsi, &s->xcn); - if (ret < 0) { - teardown_all(s); - return ret; - } - ret = init_shm(s); if (ret < 0) { teardown_all(s); return ret; } -#ifndef XRT_OS_ANDROID - ret = init_listen_socket(s); + ret = ipc_server_mainloop_init(&s->ml); if (ret < 0) { teardown_all(s); return ret; } - ret = init_epoll(s); - if (ret < 0) { - teardown_all(s); - return ret; - } -#endif - - // Init all of the render riming helpers. - for (size_t i = 0; i < ARRAY_SIZE(s->threads); i++) { - u_rt_helper_init((struct u_rt_helper *)&s->threads[i].ics.urth); - } - - ret = os_mutex_init(&s->global_state_lock); + ret = os_mutex_init(&s->global_state.lock); if (ret < 0) { teardown_all(s); return ret; @@ -606,535 +481,31 @@ init_all(struct ipc_server *s) return 0; } -static void -handle_listen(struct ipc_server *vs) -{ - int ret = accept(vs->listen_socket, NULL, NULL); - if (ret < 0) { - U_LOG_E("Error accept failed: '%i'!", ret); - vs->running = false; - } - start_client_listener_thread(vs, ret); -} - -#define NUM_POLL_EVENTS 8 -#define NO_SLEEP 0 - -static void -check_epoll(struct ipc_server *vs) -{ - int epoll_fd = vs->epoll_fd; - - struct epoll_event events[NUM_POLL_EVENTS] = {0}; - - // No sleeping, returns immediately. - int ret = epoll_wait(epoll_fd, events, NUM_POLL_EVENTS, NO_SLEEP); - if (ret < 0) { - U_LOG_E("Error epoll_wait failed: '%i'.", ret); - vs->running = false; - return; - } - - for (int i = 0; i < ret; i++) { - // If we get data on stdin, stop. - if (events[i].data.fd == 0) { - vs->running = false; - return; - } - - // Somebody new at the door. - if (events[i].data.fd == vs->listen_socket) { - handle_listen(vs); - } - } -} - -static uint32_t -find_event_slot(volatile struct ipc_client_state *ics) -{ - uint64_t oldest_event_timestamp = UINT64_MAX; - uint32_t oldest_event_index = 0; - for (uint32_t i = 0; i < IPC_EVENT_QUEUE_SIZE; i++) { - if (ics->queued_events->timestamp < oldest_event_timestamp) { - oldest_event_index = i; - } - if (!ics->queued_events[i].pending) { - return i; - } - } - - U_LOG_E("Event queue full - unconsumed event lost!"); - return oldest_event_index; -} - -static void -transition_overlay_visibility(volatile struct ipc_client_state *ics, bool visible) -{ - uint32_t event_slot = find_event_slot(ics); - uint64_t timestamp = os_monotonic_get_ns(); - - volatile struct ipc_queued_event *qe = &ics->queued_events[event_slot]; - - qe->timestamp = timestamp; - qe->pending = true; - qe->event.type = XRT_COMPOSITOR_EVENT_OVERLAY_CHANGE; - qe->event.overlay.visible = visible; -} - -static void -send_client_state(volatile struct ipc_client_state *ics) -{ - uint32_t event_slot = find_event_slot(ics); - uint64_t timestamp = os_monotonic_get_ns(); - - volatile struct ipc_queued_event *qe = &ics->queued_events[event_slot]; - - qe->timestamp = timestamp; - qe->pending = true; - qe->event.type = XRT_COMPOSITOR_EVENT_STATE_CHANGE; - qe->event.state.visible = ics->client_state.session_visible; - qe->event.state.focused = ics->client_state.session_focused; -} - -static bool -_update_projection_layer(struct xrt_compositor *xc, - volatile struct ipc_client_state *ics, - volatile struct ipc_layer_entry *layer, - uint32_t i) -{ - // xdev - uint32_t device_id = layer->xdev_id; - // left - uint32_t lxsci = layer->swapchain_ids[0]; - // right - uint32_t rxsci = layer->swapchain_ids[1]; - - struct xrt_device *xdev = get_xdev(ics, device_id); - struct xrt_swapchain *lxcs = ics->xscs[lxsci]; - struct xrt_swapchain *rxcs = ics->xscs[rxsci]; - - if (lxcs == NULL || rxcs == NULL) { - U_LOG_E("Invalid swap chain for projection layer!"); - return false; - } - - if (xdev == NULL) { - U_LOG_E("Invalid xdev for projection layer!"); - return false; - } - - // Cast away volatile. - struct xrt_layer_data *data = (struct xrt_layer_data *)&layer->data; - - xrt_comp_layer_stereo_projection(xc, xdev, lxcs, rxcs, data); - - return true; -} - -static bool -_update_projection_layer_depth(struct xrt_compositor *xc, - volatile struct ipc_client_state *ics, - volatile struct ipc_layer_entry *layer, - uint32_t i) -{ - // xdev - uint32_t xdevi = layer->xdev_id; - // left - uint32_t l_xsci = layer->swapchain_ids[0]; - // right - uint32_t r_xsci = layer->swapchain_ids[1]; - // left - uint32_t l_d_xsci = layer->swapchain_ids[2]; - // right - uint32_t r_d_xsci = layer->swapchain_ids[3]; - - struct xrt_device *xdev = get_xdev(ics, xdevi); - struct xrt_swapchain *l_xcs = ics->xscs[l_xsci]; - struct xrt_swapchain *r_xcs = ics->xscs[r_xsci]; - struct xrt_swapchain *l_d_xcs = ics->xscs[l_d_xsci]; - struct xrt_swapchain *r_d_xcs = ics->xscs[r_d_xsci]; - - if (l_xcs == NULL || r_xcs == NULL || l_d_xcs == NULL || r_d_xcs == NULL) { - U_LOG_E("Invalid swap chain for projection layer!"); - return false; - } - - if (xdev == NULL) { - U_LOG_E("Invalid xdev for projection layer!"); - return false; - } - - // Cast away volatile. - struct xrt_layer_data *data = (struct xrt_layer_data *)&layer->data; - - xrt_comp_layer_stereo_projection_depth(xc, xdev, l_xcs, r_xcs, l_d_xcs, r_d_xcs, data); - - return true; -} - -static bool -do_single(struct xrt_compositor *xc, - volatile struct ipc_client_state *ics, - volatile struct ipc_layer_entry *layer, - uint32_t i, - const char *name, - struct xrt_device **out_xdev, - struct xrt_swapchain **out_xcs, - struct xrt_layer_data **out_data) -{ - uint32_t device_id = layer->xdev_id; - uint32_t sci = layer->swapchain_ids[0]; - - struct xrt_device *xdev = get_xdev(ics, device_id); - struct xrt_swapchain *xcs = ics->xscs[sci]; - - if (xcs == NULL) { - U_LOG_E("Invalid swapchain for '%u' layer, '%s'!", i, name); - return false; - } - - if (xdev == NULL) { - U_LOG_E("Invalid xdev for '%u' layer, '%s'!", i, name); - return false; - } - - // Cast away volatile. - struct xrt_layer_data *data = (struct xrt_layer_data *)&layer->data; - - *out_xdev = xdev; - *out_xcs = xcs; - *out_data = data; - - return true; -} - -static bool -_update_quad_layer(struct xrt_compositor *xc, - volatile struct ipc_client_state *ics, - volatile struct ipc_layer_entry *layer, - uint32_t i) -{ - struct xrt_device *xdev; - struct xrt_swapchain *xcs; - struct xrt_layer_data *data; - - if (!do_single(xc, ics, layer, i, "quad", &xdev, &xcs, &data)) { - return false; - } - - xrt_comp_layer_quad(xc, xdev, xcs, data); - - return true; -} - -static bool -_update_cube_layer(struct xrt_compositor *xc, - volatile struct ipc_client_state *ics, - volatile struct ipc_layer_entry *layer, - uint32_t i) -{ - struct xrt_device *xdev; - struct xrt_swapchain *xcs; - struct xrt_layer_data *data; - - if (!do_single(xc, ics, layer, i, "cube", &xdev, &xcs, &data)) { - return false; - } - - xrt_comp_layer_cube(xc, xdev, xcs, data); - - return true; -} - -static bool -_update_cylinder_layer(struct xrt_compositor *xc, - volatile struct ipc_client_state *ics, - volatile struct ipc_layer_entry *layer, - uint32_t i) -{ - struct xrt_device *xdev; - struct xrt_swapchain *xcs; - struct xrt_layer_data *data; - - if (!do_single(xc, ics, layer, i, "cylinder", &xdev, &xcs, &data)) { - return false; - } - - xrt_comp_layer_cylinder(xc, xdev, xcs, data); - - return true; -} - -static bool -_update_equirect1_layer(struct xrt_compositor *xc, - volatile struct ipc_client_state *ics, - volatile struct ipc_layer_entry *layer, - uint32_t i) -{ - struct xrt_device *xdev; - struct xrt_swapchain *xcs; - struct xrt_layer_data *data; - - if (!do_single(xc, ics, layer, i, "equirect1", &xdev, &xcs, &data)) { - return false; - } - - xrt_comp_layer_equirect1(xc, xdev, xcs, data); - - return true; -} - -static bool -_update_equirect2_layer(struct xrt_compositor *xc, - volatile struct ipc_client_state *ics, - volatile struct ipc_layer_entry *layer, - uint32_t i) -{ - struct xrt_device *xdev; - struct xrt_swapchain *xcs; - struct xrt_layer_data *data; - - if (!do_single(xc, ics, layer, i, "equirect2", &xdev, &xcs, &data)) { - return false; - } - - xrt_comp_layer_equirect2(xc, xdev, xcs, data); - - return true; -} - -static int -_overlay_sort_func(const void *a, const void *b) -{ - struct _z_sort_data *oa = (struct _z_sort_data *)a; - struct _z_sort_data *ob = (struct _z_sort_data *)b; - if (oa->z_order < ob->z_order) { - return -1; - } - if (oa->z_order > ob->z_order) { - return 1; - } - return 0; -} - -static bool -_update_layers(struct ipc_server *s, struct xrt_compositor *xc) -{ - struct _z_sort_data z_data[IPC_MAX_CLIENTS]; - - // initialise, and fill in overlay app data - for (int32_t i = 0; i < IPC_MAX_CLIENTS; i++) { - volatile struct ipc_client_state *ics = &s->threads[i].ics; - z_data[i].index = -1; - z_data[i].z_order = -1; - // we need to create a list of overlay applications, sorted by z - if (ics->client_state.session_overlay) { - if (ics->client_state.session_active) { - z_data[i].index = i; - z_data[i].z_order = ics->client_state.z_order; - } - } - } - - // ensure our primary application is enabled, - // and rendered first in the stack - if (s->active_client_index >= 0) { - z_data[s->active_client_index].index = s->active_client_index; - z_data[s->active_client_index].z_order = INT32_MIN; - } - - // sort the stack array - qsort(z_data, IPC_MAX_CLIENTS, sizeof(struct _z_sort_data), _overlay_sort_func); - - // render the layer stack - for (uint32_t i = 0; i < IPC_MAX_CLIENTS; i++) { - struct _z_sort_data *zd = &z_data[i]; - if (zd->index < 0) { - continue; - } - - volatile struct ipc_client_state *ics = &s->threads[zd->index].ics; - - for (uint32_t j = 0; j < ics->render_state.num_layers; j++) { - volatile struct ipc_layer_entry *layer = &ics->render_state.layers[j]; - - switch (layer->data.type) { - case XRT_LAYER_STEREO_PROJECTION: - if (!_update_projection_layer(xc, ics, layer, i)) { - return false; - } - break; - case XRT_LAYER_STEREO_PROJECTION_DEPTH: - if (!_update_projection_layer_depth(xc, ics, layer, i)) { - return false; - } - break; - case XRT_LAYER_QUAD: - if (!_update_quad_layer(xc, ics, layer, i)) { - return false; - } - break; - case XRT_LAYER_CUBE: - if (!_update_cube_layer(xc, ics, layer, i)) { - return false; - } - break; - case XRT_LAYER_CYLINDER: - if (!_update_cylinder_layer(xc, ics, layer, i)) { - return false; - } - break; - case XRT_LAYER_EQUIRECT1: - if (!_update_equirect1_layer(xc, ics, layer, i)) { - return false; - } - break; - case XRT_LAYER_EQUIRECT2: - if (!_update_equirect2_layer(xc, ics, layer, i)) { - return false; - } - break; - default: U_LOG_E("Unhandled layer type '%i'!", layer->data.type); break; - } - } - } - - return true; -} - - - static int main_loop(struct ipc_server *s) { - struct xrt_compositor *xc = &s->xcn->base; - - // make sure all our client connections have a handle to the - // compositor and consistent initial state - while (s->running) { - int64_t frame_id; - uint64_t predicted_display_time_ns; - uint64_t predicted_display_period_ns; + os_nanosleep(U_TIME_1S_IN_NS / 20); - xrt_comp_wait_frame(xc, &frame_id, &predicted_display_time_ns, &predicted_display_period_ns); - - uint64_t now_ns = os_monotonic_get_ns(); - uint64_t diff_ns = predicted_display_time_ns - now_ns; - - os_mutex_lock(&s->global_state_lock); - - // Broadcast the new timing information to the helpers. - for (size_t i = 0; i < ARRAY_SIZE(s->threads); i++) { - struct u_rt_helper *urth = (struct u_rt_helper *)&s->threads[i].ics.urth; - u_rt_helper_new_sample( // - urth, // - predicted_display_time_ns, // - predicted_display_period_ns, // - diff_ns); // - } - - os_mutex_unlock(&s->global_state_lock); - - - xrt_comp_begin_frame(xc, frame_id); - xrt_comp_layer_begin(xc, frame_id, 0); - - _update_layers(s, xc); - - xrt_comp_layer_commit(xc, frame_id, XRT_GRAPHICS_SYNC_HANDLE_INVALID); - -#ifndef XRT_OS_ANDROID - // Check polling last, so we know we have valid timing data. - check_epoll(s); -#endif + // Check polling. + ipc_server_mainloop_poll(s, &s->ml); } return 0; } - static void -handle_overlay_client_events(volatile struct ipc_client_state *ics, int active_id, int prev_active_id) -{ - // this is an overlay session. - if (ics->client_state.session_overlay) { - - // switch between main applications - if (active_id >= 0 && prev_active_id >= 0) { - transition_overlay_visibility(ics, false); - transition_overlay_visibility(ics, true); - } - - // switch from idle to active application - if (active_id >= 0 && prev_active_id < 0) { - transition_overlay_visibility(ics, true); - } - - // switch from active application to idle - if (active_id < 0 && prev_active_id >= 0) { - transition_overlay_visibility(ics, false); - } - } -} - -static void -handle_focused_client_events(volatile struct ipc_client_state *ics, int active_id, int prev_active_id) -{ - - // if our prev active id is -1 and our cur active id is -1, we - // can bail out early - - if (active_id == -1 && prev_active_id == -1) { - return; - } - - // set visibility/focus to false on all applications - ics->client_state.session_focused = false; - ics->client_state.session_visible = false; - - // do we have a primary application? - if (active_id >= 0) { - - // if we are an overlay, we are always visible - // if we have a primary application - if (ics->client_state.session_overlay) { - ics->client_state.session_visible = true; - } - - // set visible + focused if we are the primary - // application - if (ics->server_thread_index == active_id) { - ics->client_state.session_visible = true; - ics->client_state.session_focused = true; - } - send_client_state(ics); - return; - } - - // no primary application, set all overlays to synchronised - // state - if (ics->client_state.session_overlay) { - ics->client_state.session_focused = false; - ics->client_state.session_visible = false; - send_client_state(ics); - } -} - -void init_server_state(struct ipc_server *s) { - // set up initial state for global vars, and each client state - s->active_client_index = -1; // we start off with no active client. - s->last_active_client_index = -1; + s->global_state.active_client_index = -1; // we start off with no active client. + s->global_state.last_active_client_index = -1; s->current_slot_index = 0; for (uint32_t i = 0; i < IPC_MAX_CLIENTS; i++) { volatile struct ipc_client_state *ics = &s->threads[i].ics; ics->server = s; - ics->xc = &s->xcn->base; ics->server_thread_index = -1; } } @@ -1142,26 +513,105 @@ init_server_state(struct ipc_server *s) /* * - * Exported functions. + * Client management functions. * */ -void -update_server_state(struct ipc_server *s) +static void +handle_overlay_client_events(volatile struct ipc_client_state *ics, int active_id, int prev_active_id) { - // multiple threads could call this at the same time. - os_mutex_lock(&s->global_state_lock); + // Is an overlay session? + if (!ics->client_state.session_overlay) { + return; + } + // Does this client have a compositor yet, if not return? + if (ics->xc == NULL) { + return; + } + + // Switch between main applications + if (active_id >= 0 && prev_active_id >= 0) { + xrt_syscomp_set_main_app_visibility(ics->server->xsysc, ics->xc, false); + xrt_syscomp_set_main_app_visibility(ics->server->xsysc, ics->xc, true); + } + + // Switch from idle to active application + if (active_id >= 0 && prev_active_id < 0) { + xrt_syscomp_set_main_app_visibility(ics->server->xsysc, ics->xc, true); + } + + // Switch from active application to idle + if (active_id < 0 && prev_active_id >= 0) { + xrt_syscomp_set_main_app_visibility(ics->server->xsysc, ics->xc, false); + } +} + +static void +handle_focused_client_events(volatile struct ipc_client_state *ics, int active_id, int prev_active_id) +{ + // Set start z_order at the bottom. + int64_t z_order = INT64_MIN; + + // Set visibility/focus to false on all applications. + bool focused = false; + bool visible = false; + + // Set visible + focused if we are the primary application + if (ics->server_thread_index == active_id) { + visible = true; + focused = true; + z_order = INT64_MIN; + } + + // Set all overlays to always active and focused. + if (ics->client_state.session_overlay) { + visible = true; + focused = true; + z_order = ics->client_state.z_order; + } + + ics->client_state.session_visible = visible; + ics->client_state.session_focused = focused; + ics->client_state.z_order = z_order; + + if (ics->xc != NULL) { + xrt_syscomp_set_state(ics->server->xsysc, ics->xc, visible, focused); + xrt_syscomp_set_z_order(ics->server->xsysc, ics->xc, z_order); + } +} + +static void +flush_state_to_all_clients_locked(struct ipc_server *s) +{ + for (uint32_t i = 0; i < IPC_MAX_CLIENTS; i++) { + volatile struct ipc_client_state *ics = &s->threads[i].ics; + + // Not running? + if (ics->server_thread_index < 0) { + continue; + } + + handle_focused_client_events(ics, s->global_state.active_client_index, + s->global_state.last_active_client_index); + handle_overlay_client_events(ics, s->global_state.active_client_index, + s->global_state.last_active_client_index); + } +} + +static void +update_server_state_locked(struct ipc_server *s) +{ // if our client that is set to active is still active, // and it is the same as our last active client, we can // early-out, as no events need to be sent - if (s->active_client_index >= 0) { + if (s->global_state.active_client_index >= 0) { - volatile struct ipc_client_state *ics = &s->threads[s->active_client_index].ics; + volatile struct ipc_client_state *ics = &s->threads[s->global_state.active_client_index].ics; - if (ics->client_state.session_active && s->active_client_index == s->last_active_client_index) { - os_mutex_unlock(&s->global_state_lock); + if (ics->client_state.session_active && + s->global_state.active_client_index == s->global_state.last_active_client_index) { return; } } @@ -1191,83 +641,154 @@ update_server_state(struct ipc_server *s) // if our currently-set active primary application is not // actually active/displayable, use the fallback application // instead. - volatile struct ipc_client_state *ics = &s->threads[s->active_client_index].ics; - if (!(ics->client_state.session_overlay == false && s->active_client_index >= 0 && + volatile struct ipc_client_state *ics = &s->threads[s->global_state.active_client_index].ics; + if (!(ics->client_state.session_overlay == false && s->global_state.active_client_index >= 0 && ics->client_state.session_active)) { - s->active_client_index = fallback_active_application; + s->global_state.active_client_index = fallback_active_application; } // if we have no applications to fallback to, enable the idle // wallpaper. if (set_idle) { - s->active_client_index = -1; + s->global_state.active_client_index = -1; } - for (uint32_t i = 0; i < IPC_MAX_CLIENTS; i++) { + flush_state_to_all_clients_locked(s); - volatile struct ipc_client_state *ics = &s->threads[i].ics; - if (ics->server_thread_index >= 0) { - - handle_focused_client_events(ics, s->active_client_index, s->last_active_client_index); - - handle_overlay_client_events(ics, s->active_client_index, s->last_active_client_index); - } - } - - s->last_active_client_index = s->active_client_index; - - os_mutex_unlock(&s->global_state_lock); + s->global_state.last_active_client_index = s->global_state.active_client_index; } + +/* + * + * Exported functions. + * + */ + +void +ipc_server_set_active_client(struct ipc_server *s, int client_id) +{ + os_mutex_lock(&s->global_state.lock); + + if (client_id == s->global_state.active_client_index) { + os_mutex_unlock(&s->global_state.lock); + return; + } + + + + os_mutex_unlock(&s->global_state.lock); +} + +void +ipc_server_activate_session(volatile struct ipc_client_state *ics) +{ + struct ipc_server *s = ics->server; + + // Already active, noop. + if (ics->client_state.session_active) { + return; + } + + assert(ics->server_thread_index >= 0); + + // Multiple threads could call this at the same time. + os_mutex_lock(&s->global_state.lock); + + ics->client_state.session_active = true; + + if (ics->client_state.session_overlay) { + // For new active overlay sessions only update this session. + handle_focused_client_events(ics, s->global_state.active_client_index, + s->global_state.last_active_client_index); + handle_overlay_client_events(ics, s->global_state.active_client_index, + s->global_state.last_active_client_index); + } else { + // For new active regular sessions update all clients. + update_server_state_locked(s); + } + + os_mutex_unlock(&s->global_state.lock); +} + +void +ipc_server_deactivate_session(volatile struct ipc_client_state *ics) +{ + struct ipc_server *s = ics->server; + + // Multiple threads could call this at the same time. + os_mutex_lock(&s->global_state.lock); + + ics->client_state.session_active = false; + + update_server_state_locked(s); + + os_mutex_unlock(&s->global_state.lock); +} + +void +ipc_server_update_state(struct ipc_server *s) +{ + // Multiple threads could call this at the same time. + os_mutex_lock(&s->global_state.lock); + + update_server_state_locked(s); + + os_mutex_unlock(&s->global_state.lock); +} + +#ifndef XRT_OS_ANDROID int ipc_server_main(int argc, char **argv) { struct ipc_server *s = U_TYPED_CALLOC(struct ipc_server); -#ifndef XRT_OS_ANDROID /* ---- HACK ---- */ // need to create early before any vars are added oxr_sdl2_hack_create(&s->hack); /* ---- HACK ---- */ -#endif int ret = init_all(s); if (ret < 0) { + free(s->hack); free(s); return ret; } init_server_state(s); -#ifndef XRT_OS_ANDROID + struct xrt_device *xdevs[IPC_SERVER_NUM_XDEVS]; + for (size_t i = 0; i < IPC_SERVER_NUM_XDEVS; i++) { + xdevs[i] = s->idevs[i].xdev; + } + /* ---- HACK ---- */ - oxr_sdl2_hack_start(s->hack, s->xinst); + oxr_sdl2_hack_start(s->hack, s->xinst, xdevs); /* ---- HACK ---- */ -#endif ret = main_loop(s); -#ifndef XRT_OS_ANDROID /* ---- HACK ---- */ oxr_sdl2_hack_stop(&s->hack); /* ---- HACK ---- */ -#endif teardown_all(s); free(s); - U_LOG_E("Server exiting: '%i'!", ret); + U_LOG_I("Server exiting: '%i'!", ret); return ret; } +#endif // !XRT_OS_ANDROID + #ifdef XRT_OS_ANDROID int -ipc_server_main_android(int fd) +ipc_server_main_android(struct ipc_server **ps, void (*startup_complete_callback)(void *data), void *data) { struct ipc_server *s = U_TYPED_CALLOC(struct ipc_server); - U_LOG_D("Created IPC server on fd '%d'!", fd); + U_LOG_D("Created IPC server!"); int ret = init_all(s); if (ret < 0) { @@ -1276,14 +797,17 @@ ipc_server_main_android(int fd) } init_server_state(s); - start_client_listener_thread(s, fd); + + *ps = s; + startup_complete_callback(data); + ret = main_loop(s); teardown_all(s); free(s); - U_LOG_E("Server exiting '%i'!", ret); + U_LOG_I("Server exiting '%i'!", ret); return ret; } -#endif +#endif // XRT_OS_ANDROID diff --git a/src/xrt/ipc/shared/ipc_protocol.h b/src/xrt/ipc/shared/ipc_protocol.h index c4c365d3d..6bac624b5 100644 --- a/src/xrt/ipc/shared/ipc_protocol.h +++ b/src/xrt/ipc/shared/ipc_protocol.h @@ -20,7 +20,7 @@ #include "xrt/xrt_tracking.h" -#define IPC_MSG_SOCK_FILE "/tmp/monado_comp_ipc" +#define IPC_MSG_SOCK_FILE "monado_comp_ipc" #define IPC_MAX_SWAPCHAIN_HANDLES 8 #define IPC_CRED_SIZE 1 // auth not implemented #define IPC_BUF_SIZE 512 // must be >= largest message length in bytes @@ -37,6 +37,8 @@ #define IPC_SHARED_MAX_OUTPUTS 128 #define IPC_SHARED_MAX_BINDINGS 64 +// example: v21.0.0-560-g586d33b5 +#define IPC_VERSION_NAME_LEN 64 /* * @@ -151,6 +153,7 @@ struct ipc_layer_entry */ struct ipc_layer_slot { + uint64_t display_time_ns; enum xrt_blend_mode env_blend_mode; uint32_t num_layers; struct ipc_layer_entry layers[IPC_MAX_LAYERS]; @@ -173,6 +176,11 @@ struct ipc_layer_slot */ struct ipc_shared_memory { + /*! + * The git revision of the service, used by clients to detect version mismatches. + */ + char u_git_tag[IPC_VERSION_NAME_LEN]; + /*! * Number of elements in @ref itracks that are populated/valid. */ @@ -224,6 +232,8 @@ struct ipc_shared_memory */ struct xrt_fov fov; } views[2]; + enum xrt_blend_mode blend_modes[XRT_MAX_DEVICE_BLEND_MODES]; + size_t num_blend_modes; } hmd; struct xrt_input inputs[IPC_SHARED_MAX_INPUTS]; diff --git a/src/xrt/ipc/shared/ipc_utils.h b/src/xrt/ipc/shared/ipc_utils.h index 49a13f9ba..79069e8a7 100644 --- a/src/xrt/ipc/shared/ipc_utils.h +++ b/src/xrt/ipc/shared/ipc_utils.h @@ -137,7 +137,7 @@ ipc_send_fds(struct ipc_message_channel *imc, const void *data, size_t size, con * must be greater than 0 and must match the value provided at the other end. * * @public @memberof ipc_message_channel - * @relatesalso xrt_shmem_handle_t + * @see xrt_shmem_handle_t */ xrt_result_t ipc_receive_handles_shmem(struct ipc_message_channel *imc, @@ -161,7 +161,7 @@ ipc_receive_handles_shmem(struct ipc_message_channel *imc, * time, because the receiver must have the same value in its receive call. * * @public @memberof ipc_message_channel - * @relatesalso xrt_shmem_handle_t + * @see xrt_shmem_handle_t */ xrt_result_t ipc_send_handles_shmem(struct ipc_message_channel *imc, @@ -195,7 +195,7 @@ ipc_send_handles_shmem(struct ipc_message_channel *imc, * must be greater than 0 and must match the value provided at the other end. * * @public @memberof ipc_message_channel - * @relatesalso xrt_graphics_buffer_handle_t + * @see xrt_graphics_buffer_handle_t */ xrt_result_t ipc_receive_handles_graphics_buffer(struct ipc_message_channel *imc, @@ -220,7 +220,7 @@ ipc_receive_handles_graphics_buffer(struct ipc_message_channel *imc, * time, because the receiver must have the same value in its receive call. * * @public @memberof ipc_message_channel - * @relatesalso xrt_graphics_buffer_handle_t + * @see xrt_graphics_buffer_handle_t */ xrt_result_t ipc_send_handles_graphics_buffer(struct ipc_message_channel *imc, @@ -254,7 +254,7 @@ ipc_send_handles_graphics_buffer(struct ipc_message_channel *imc, * must be greater than 0 and must match the value provided at the other end. * * @public @memberof ipc_message_channel - * @relatesalso xrt_graphics_sync_handle_t + * @see xrt_graphics_sync_handle_t */ xrt_result_t ipc_receive_handles_graphics_sync(struct ipc_message_channel *imc, @@ -277,7 +277,7 @@ ipc_receive_handles_graphics_sync(struct ipc_message_channel *imc, * because the receiver must have the same value in its receive call. * * @public @memberof ipc_message_channel - * @relatesalso xrt_graphics_sync_handle_t + * @see xrt_graphics_sync_handle_t */ xrt_result_t ipc_send_handles_graphics_sync(struct ipc_message_channel *imc, diff --git a/src/xrt/ipc/shared/ipcproto/common.py b/src/xrt/ipc/shared/ipcproto/common.py index 9c4d50f3b..3bb85afb7 100644 --- a/src/xrt/ipc/shared/ipcproto/common.py +++ b/src/xrt/ipc/shared/ipcproto/common.py @@ -55,7 +55,8 @@ class Arg: # Keep all these synchronized with the definitions in the JSON Schema. SCALAR_TYPES = set(("uint32_t", "int64_t", - "uint64_t")) + "uint64_t", + "bool")) AGGREGATE_RE = re.compile(r"((const )?struct|union) (xrt|ipc)_[a-z_]+") ENUM_RE = re.compile(r"enum xrt_[a-z_]+") diff --git a/src/xrt/ipc/shared/proto.json b/src/xrt/ipc/shared/proto.json index 9da7a32c3..8df780c8b 100644 --- a/src/xrt/ipc/shared/proto.json +++ b/src/xrt/ipc/shared/proto.json @@ -66,19 +66,20 @@ "session_end": {}, + "session_destroy": {}, + "compositor_get_info": { "out": [ {"name": "info", "type": "struct xrt_compositor_info"} ] }, - "compositor_wait_frame": { + "compositor_predict_frame": { "out": [ {"name": "frame_id", "type": "int64_t"}, - {"name": "predicted_display_time", "type": "uint64_t"}, {"name": "wake_up_time", "type": "uint64_t"}, - {"name": "predicted_display_period", "type": "uint64_t"}, - {"name": "min_display_period", "type": "uint64_t"} + {"name": "predicted_display_time", "type": "uint64_t"}, + {"name": "predicted_display_period", "type": "uint64_t"} ] }, @@ -124,7 +125,8 @@ "out": [ {"name": "id", "type": "uint32_t"}, {"name": "num_images", "type": "uint32_t"}, - {"name": "size", "type": "uint64_t"} + {"name": "size", "type": "uint64_t"}, + {"name": "use_dedicated_allocation", "type": "bool"} ], "out_handles": {"type": "xrt_graphics_buffer_handle_t"} }, @@ -194,14 +196,15 @@ {"name": "at_timestamp", "type": "uint64_t"} ], "out": [ - {"name": "value", "type": "struct xrt_hand_joint_set"} + {"name": "value", "type": "struct xrt_hand_joint_set"}, + {"name": "timestamp", "type": "uint64_t"} ] }, "device_get_view_pose": { "in": [ {"name": "id", "type": "uint32_t"}, - {"name": "eye_relation", "type": "struct xrt_vec3"}, + {"name": "eye_relation", "type": "const struct xrt_vec3"}, {"name": "view_index", "type": "uint32_t"} ], "out": [ diff --git a/src/xrt/ipc/shared/proto.schema.json b/src/xrt/ipc/shared/proto.schema.json index ee044fa95..c291edabc 100644 --- a/src/xrt/ipc/shared/proto.schema.json +++ b/src/xrt/ipc/shared/proto.schema.json @@ -12,7 +12,8 @@ "enum": [ "uint32_t", "int64_t", - "uint64_t" + "uint64_t", + "bool" ] }, "aggregate": { diff --git a/src/xrt/state_trackers/gui/CMakeLists.txt b/src/xrt/state_trackers/gui/CMakeLists.txt index ac28c50ef..be855163e 100644 --- a/src/xrt/state_trackers/gui/CMakeLists.txt +++ b/src/xrt/state_trackers/gui/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2019-2020, Collabora, Ltd. +# Copyright 2019-2021, Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 # c-imgui doesn't do well with IPO - lots of warnings. @@ -13,8 +13,13 @@ set(GUI_SOURCE_FILES gui_scene_calibrate.c gui_scene_debug.c gui_scene_main_menu.c + gui_scene_record.c gui_scene_remote.c gui_scene_video.c + gui_scene_tracking_overrides.c + gui_stb.c + gui_window_record.c + gui_window_record.h ../../../external/imgui/imgui/cimgui.cpp ../../../external/imgui/imgui/cimgui.h ../../../external/imgui/imgui/cimplot.cpp @@ -30,6 +35,7 @@ set(GUI_SOURCE_FILES ../../../external/imgui/imgui/imgui_widgets.cpp ../../../external/imgui/imgui/implot.cpp ../../../external/imgui/imgui/implot.h + ../../../external/imgui/imgui/implot_demo.cpp ../../../external/imgui/imgui/implot_internal.h ../../../external/imgui/imgui/implot_items.cpp ../../../external/imgui/imgui/imstb_rectpack.h @@ -44,6 +50,7 @@ add_library(st_gui STATIC target_link_libraries(st_gui PRIVATE xrt-external-glad + xrt-external-stb aux_util aux_os ) @@ -56,6 +63,18 @@ target_compile_definitions(st_gui PUBLIC CIMGUI_NO_EXPORT ) +if(XRT_HAVE_GST) + target_link_libraries(st_gui PRIVATE + aux_gstreamer + ) +endif() + +if(XRT_BUILD_DRIVER_DEPTHAI) + target_link_libraries(st_gui PRIVATE + drv_depthai + ) +endif() + if(XRT_BUILD_DRIVER_REMOTE) target_link_libraries(st_gui PRIVATE drv_remote diff --git a/src/xrt/state_trackers/gui/gui_common.h b/src/xrt/state_trackers/gui/gui_common.h index 015e3db02..3405b7ddb 100644 --- a/src/xrt/state_trackers/gui/gui_common.h +++ b/src/xrt/state_trackers/gui/gui_common.h @@ -197,6 +197,14 @@ gui_scene_main_menu(struct gui_program *p); void gui_scene_select_video_calibrate(struct gui_program *p); +/*! + * Shows a UI that lets you set up tracking overrides. + * + * @ingroup gui + */ +void +gui_scene_tracking_overrides(struct gui_program *p); + /*! * Regular debug UI. * @@ -205,6 +213,14 @@ gui_scene_select_video_calibrate(struct gui_program *p); void gui_scene_debug(struct gui_program *p); +/*! + * Create a recording view scene. + * + * @ingroup gui + */ +void +gui_scene_record(struct gui_program *p, const char *camera); + /*! * Remote control debugging UI. * diff --git a/src/xrt/state_trackers/gui/gui_ogl.c b/src/xrt/state_trackers/gui/gui_ogl.c index cce444977..51f402af6 100644 --- a/src/xrt/state_trackers/gui/gui_ogl.c +++ b/src/xrt/state_trackers/gui/gui_ogl.c @@ -61,6 +61,9 @@ break_apart(struct xrt_frame_node *node) pthread_mutex_lock(&s->mutex); s->running = false; pthread_mutex_unlock(&s->mutex); + + // Release any frame waiting for upload. + xrt_frame_reference(&s->frame, NULL); } static void @@ -170,6 +173,8 @@ gui_ogl_sink_create(const char *name, struct xrt_frame_context *xfctx, struct xr glBindTexture(GL_TEXTURE_2D, 0); + xrt_frame_context_add(xfctx, &s->node); + *out_sink = &s->sink; return &s->tex; diff --git a/src/xrt/state_trackers/gui/gui_scene_calibrate.c b/src/xrt/state_trackers/gui/gui_scene_calibrate.c index 1beed1c86..38a13c321 100644 --- a/src/xrt/state_trackers/gui/gui_scene_calibrate.c +++ b/src/xrt/state_trackers/gui/gui_scene_calibrate.c @@ -13,6 +13,7 @@ #include "util/u_sink.h" #include "util/u_file.h" #include "util/u_json.h" +#include "util/u_config_json.h" #ifdef XRT_HAVE_OPENCV #include "tracking/t_tracking.h" @@ -102,31 +103,10 @@ save_calibration(struct calibration_scene *cs) * Camera config file. * */ - - cJSON *root = cJSON_CreateObject(); - cJSON *t = cJSON_AddObjectToObject(root, "tracking"); - cJSON_AddNumberToObject(t, "version", 0); - cJSON_AddStringToObject(t, "camera_name", cs->settings->camera_name); - cJSON_AddNumberToObject(t, "camera_mode", cs->settings->camera_mode); - switch (cs->settings->camera_type) { - case XRT_SETTINGS_CAMERA_TYPE_REGULAR_MONO: cJSON_AddStringToObject(t, "camera_type", "regular_mono"); break; - case XRT_SETTINGS_CAMERA_TYPE_REGULAR_SBS: cJSON_AddStringToObject(t, "camera_type", "regular_sbs"); break; - case XRT_SETTINGS_CAMERA_TYPE_PS4: cJSON_AddStringToObject(t, "camera_type", "ps4"); break; - case XRT_SETTINGS_CAMERA_TYPE_LEAP_MOTION: cJSON_AddStringToObject(t, "camera_type", "leap_motion"); break; - } - cJSON_AddStringToObject(t, "calibration_path", cs->settings->calibration_path); - - char *str = cJSON_Print(root); - U_LOG_D("%s", str); - cJSON_Delete(root); - - FILE *config_file = u_file_open_file_in_config_dir("config_v0.json", "w"); - fprintf(config_file, "%s\n", str); - fflush(config_file); - fclose(config_file); - config_file = NULL; - free(str); - + struct u_config_json json; + u_config_json_open_or_create_main_file(&json); + u_config_json_save_calibration(&json, cs->settings); + u_config_json_close(&json); /* * @@ -293,7 +273,7 @@ scene_render_select(struct gui_scene *scene, struct gui_program *p) igInputInt("Collect in groups of #", &cs->params.num_collect_restart, 1, 5, 0); igSeparator(); - igComboStr("Board type", (int *)&cs->params.pattern, "Checkers\0Circles\0Asymetric Circles\0\0", 3); + igComboStr("Board type", (int *)&cs->params.pattern, "Checkers\0Corners SB\0Circles\0Asymetric Circles\0\0", 3); switch (cs->params.pattern) { case T_BOARD_CHECKERS: igInputInt("Checkerboard Rows", &cs->params.checkers.rows, 1, 5, 0); @@ -302,6 +282,13 @@ scene_render_select(struct gui_scene *scene, struct gui_program *p) igCheckbox("Subpixel", &cs->params.checkers.subpixel_enable); igInputInt("Subpixel Search Size", &cs->params.checkers.subpixel_size, 1, 5, 0); break; + case T_BOARD_SB_CHECKERS: + igInputInt("Internal Corner Rows", &cs->params.sb_checkers.rows, 1, 5, 0); + igInputInt("Internal Corner Columns", &cs->params.sb_checkers.cols, 1, 5, 0); + igInputFloat("Corner Spacing (m)", &cs->params.sb_checkers.size_meters, 0.0005, 0.001, NULL, 0); + igCheckbox("Marker", &cs->params.sb_checkers.marker); + igCheckbox("Normalize image", &cs->params.sb_checkers.normalize_image); + break; case T_BOARD_CIRCLES: igInputInt("Circle Rows", &cs->params.circles.rows, 1, 5, 0); igInputInt("Circle Columns", &cs->params.circles.cols, 1, 5, 0); diff --git a/src/xrt/state_trackers/gui/gui_scene_debug.c b/src/xrt/state_trackers/gui/gui_scene_debug.c index ff5dd712d..7eab7af21 100644 --- a/src/xrt/state_trackers/gui/gui_scene_debug.c +++ b/src/xrt/state_trackers/gui/gui_scene_debug.c @@ -30,11 +30,19 @@ #include "gui_common.h" #include "gui_imgui.h" +#include "gui_window_record.h" #include "imgui_monado/cimgui_monado.h" #include +struct debug_record +{ + void *ptr; + + struct gui_record_window rw; +}; + /*! * A GUI scene showing the variable tracking provided by @ref util/u_var.h * @implements gui_scene @@ -43,6 +51,15 @@ struct debug_scene { struct gui_scene base; struct xrt_frame_context *xfctx; + + struct debug_record recs[32]; + uint32_t num_recrs; +}; + +struct priv_tuple +{ + struct gui_program *p; + struct debug_scene *ds; }; @@ -97,6 +114,7 @@ handle_draggable_quat(const char *name, struct xrt_quat *q) struct draw_state { struct gui_program *p; + struct debug_scene *ds; bool hidden; }; @@ -130,14 +148,16 @@ on_ff_vec3_var(struct u_var_info *info, struct gui_program *p) struct xrt_vec3 value = {0}; + uint64_t timestamp; m_ff_vec3_f32_get(ff, 0, &value, ×tamp); + float value_arr[3] = {value.x, value.y, value.z}; snprintf(tmp, sizeof(tmp), "%s.toggle", name); igToggleButton(tmp, &info->gui.graphed); igSameLine(0, 0); - igInputFloat3(name, &value.x, "%+f", ImGuiInputTextFlags_ReadOnly); + igInputFloat3(name, value_arr, "%+f", ImGuiInputTextFlags_ReadOnly); if (!info->gui.graphed) { return; @@ -170,16 +190,12 @@ on_ff_vec3_var(struct u_var_info *info, struct gui_program *p) } static void -on_sink_var(const char *name, void *ptr, struct gui_program *p) +on_sink_debug_var(const char *name, void *ptr, struct gui_program *p, struct debug_scene *ds) { - for (size_t i = 0; i < ARRAY_SIZE(p->texs); i++) { - struct gui_ogl_texture *tex = p->texs[i]; + for (size_t i = 0; i < ARRAY_SIZE(ds->recs); i++) { + struct debug_record *dr = &ds->recs[i]; - if (tex == NULL) { - continue; - } - - if ((ptrdiff_t)tex->ptr != (ptrdiff_t)ptr) { + if ((ptrdiff_t)dr->ptr != (ptrdiff_t)ptr) { continue; } @@ -187,24 +203,40 @@ on_sink_var(const char *name, void *ptr, struct gui_program *p) continue; } - gui_ogl_sink_update(tex); - - igText("Sequence %u", (uint32_t)tex->seq); - char temp[512]; - snprintf(temp, 512, "Half (%s)", tex->name); - igCheckbox(temp, &tex->half); - int w = tex->w / (tex->half ? 2 : 1); - int h = tex->h / (tex->half ? 2 : 1); - - ImVec2 size = {(float)w, (float)h}; - ImVec2 uv0 = {0, 0}; - ImVec2 uv1 = {1, 1}; - ImVec4 white = {1, 1, 1, 1}; - ImTextureID id = (ImTextureID)(intptr_t)tex->id; - igImage(id, size, uv0, uv1, white, white); + gui_window_record_render(&dr->rw, p); } } +static void +on_button_var(const char *name, void *ptr) +{ + struct u_var_button *btn = (struct u_var_button *)ptr; + ImVec2 dims = {btn->width, btn->height}; + const char *label = strlen(btn->label) == 0 ? name : btn->label; + bool disabled = btn->disabled; + + if (disabled) { + igPushStyleVarFloat(ImGuiStyleVar_Alpha, 0.6f); + igPushItemFlag(ImGuiItemFlags_Disabled, true); + } + + if (igButton(label, dims)) { + btn->cb(btn->ptr); + } + + if (disabled) { + igPopItemFlag(); + igPopStyleVar(1); + } +} + +static void +on_draggable_f32_var(const char *name, void *ptr) +{ + struct u_var_draggable_f32 *d = (struct u_var_draggable_f32 *)ptr; + igDragFloat(name, &d->val, d->step, d->min, d->max, "%+f", ImGuiSliderFlags_None); +} + static void on_root_enter(const char *name, void *priv) { @@ -259,9 +291,11 @@ on_elem(struct u_var_info *info, void *priv) break; } case U_VAR_KIND_U8: igDragScalar(name, ImGuiDataType_U8, ptr, drag_speed, NULL, NULL, NULL, power); break; + case U_VAR_KIND_U64: igDragScalar(name, ImGuiDataType_U64, ptr, drag_speed, NULL, NULL, NULL, power); break; case U_VAR_KIND_I32: igInputInt(name, (int *)ptr, 1, 10, i_flags); break; case U_VAR_KIND_VEC3_I32: igInputInt3(name, (int *)ptr, i_flags); break; case U_VAR_KIND_F32: igInputFloat(name, (float *)ptr, 1, 10, "%+f", i_flags); break; + case U_VAR_KIND_F64: igInputDouble(name, (double *)ptr, 0.1, 1, "%+f", i_flags); break; case U_VAR_KIND_F32_ARR: { struct u_var_f32_arr *f32_arr = ptr; int index = *f32_arr->index_ptr; @@ -330,7 +364,9 @@ on_elem(struct u_var_info *info, void *priv) state->hidden = !igCollapsingHeaderBoolPtr(name, NULL, 0); break; } - case U_VAR_KIND_SINK: on_sink_var(name, ptr, state->p); break; + case U_VAR_KIND_SINK_DEBUG: on_sink_debug_var(name, ptr, state->p, state->ds); break; + case U_VAR_KIND_DRAGGABLE_F32: on_draggable_f32_var(name, ptr); break; + case U_VAR_KIND_BUTTON: on_button_var(name, ptr); break; default: igLabelText(name, "Unknown tag '%i'", kind); break; } } @@ -344,29 +380,6 @@ on_root_exit(const char *name, void *priv) igEnd(); } -static void -scene_render(struct gui_scene *scene, struct gui_program *p) -{ - struct debug_scene *ds = (struct debug_scene *)scene; - (void)ds; - struct draw_state state = {p, false}; - - u_var_visit(on_root_enter, on_root_exit, on_elem, &state); -} - -static void -scene_destroy(struct gui_scene *scene, struct gui_program *p) -{ - struct debug_scene *ds = (struct debug_scene *)scene; - - if (ds->xfctx != NULL) { - xrt_frame_context_destroy_nodes(ds->xfctx); - ds->xfctx = NULL; - } - - free(ds); -} - /* * @@ -379,14 +392,13 @@ on_root_enter_sink(const char *name, void *priv) {} static void -on_elem_sink(struct u_var_info *info, void *priv) +on_elem_sink_debug_add(struct u_var_info *info, void *priv) { - const char *name = info->name; void *ptr = info->ptr; enum u_var_kind kind = info->kind; - struct gui_program *p = (struct gui_program *)priv; + struct gui_program *p = ((struct priv_tuple *)priv)->p; - if (kind != U_VAR_KIND_SINK) { + if (kind != U_VAR_KIND_SINK_DEBUG) { return; } @@ -394,20 +406,28 @@ on_elem_sink(struct u_var_info *info, void *priv) return; } - struct xrt_frame_context *xfctx = p->xp->tracking->xfctx; - struct xrt_frame_sink **xsink_ptr = (struct xrt_frame_sink **)ptr; - struct xrt_frame_sink *split = NULL; + struct u_sink_debug *usd = (struct u_sink_debug *)ptr; + struct debug_scene *ds = ((struct priv_tuple *)priv)->ds; + struct debug_record *dr = &ds->recs[ds->num_recrs++]; - p->texs[p->num_texs] = gui_ogl_sink_create(name, xfctx, &split); - p->texs[p->num_texs++]->ptr = ptr; + dr->ptr = ptr; - u_sink_create_to_r8g8b8_or_l8(xfctx, split, &split); + gui_window_record_init(&dr->rw); + u_sink_debug_set_sink(usd, &dr->rw.sink); +} - if (*xsink_ptr != NULL) { - u_sink_split_create(xfctx, split, *xsink_ptr, xsink_ptr); - } else { - *xsink_ptr = split; +static void +on_elem_sink_debug_remove(struct u_var_info *info, void *priv) +{ + void *ptr = info->ptr; + enum u_var_kind kind = info->kind; + + if (kind != U_VAR_KIND_SINK_DEBUG) { + return; } + + struct u_sink_debug *usd = (struct u_sink_debug *)ptr; + u_sink_debug_set_sink(usd, NULL); } static void @@ -415,6 +435,39 @@ on_root_exit_sink(const char *name, void *priv) {} +/* + * + * Scene functions. + * + */ + +static void +scene_render(struct gui_scene *scene, struct gui_program *p) +{ + struct debug_scene *ds = (struct debug_scene *)scene; + struct draw_state state = {p, ds, false}; + + u_var_visit(on_root_enter, on_root_exit, on_elem, &state); +} + +static void +scene_destroy(struct gui_scene *scene, struct gui_program *p) +{ + struct debug_scene *ds = (struct debug_scene *)scene; + + // Remove the sink interceptors. + struct priv_tuple pt = {p, ds}; + u_var_visit(on_root_enter_sink, on_root_exit_sink, on_elem_sink_debug_remove, &pt); + + if (ds->xfctx != NULL) { + xrt_frame_context_destroy_nodes(ds->xfctx); + ds->xfctx = NULL; + } + + free(ds); +} + + /* * * 'Exported' functions. @@ -432,5 +485,6 @@ gui_scene_debug(struct gui_program *p) gui_scene_push_front(p, &ds->base); // Create the sink interceptors. - u_var_visit(on_root_enter_sink, on_root_exit_sink, on_elem_sink, p); + struct priv_tuple pt = {p, ds}; + u_var_visit(on_root_enter_sink, on_root_exit_sink, on_elem_sink_debug_add, &pt); } diff --git a/src/xrt/state_trackers/gui/gui_scene_main_menu.c b/src/xrt/state_trackers/gui/gui_scene_main_menu.c index 2f0d70b8e..356bc0193 100644 --- a/src/xrt/state_trackers/gui/gui_scene_main_menu.c +++ b/src/xrt/state_trackers/gui/gui_scene_main_menu.c @@ -34,6 +34,12 @@ scene_render(struct gui_scene *scene, struct gui_program *p) gui_scene_select_video_calibrate(p); } + if (igButton("Tracking Overrides", button_dims)) { + gui_scene_delete_me(p, scene); + + gui_scene_tracking_overrides(p); + } + if (igButton("Debug Test", button_dims)) { gui_scene_delete_me(p, scene); @@ -45,6 +51,21 @@ scene_render(struct gui_scene *scene, struct gui_program *p) gui_scene_debug(p); } + if (igButton("Record (DepthAI)", button_dims)) { + gui_scene_delete_me(p, scene); + gui_scene_record(p, "depthai"); + } + + if (igButton("Record (Index)", button_dims)) { + gui_scene_delete_me(p, scene); + gui_scene_record(p, "index"); + } + + if (igButton("Record (Leap Motion)", button_dims)) { + gui_scene_delete_me(p, scene); + gui_scene_record(p, "leap_motion"); + } + if (igButton("Remote", button_dims)) { gui_scene_delete_me(p, scene); diff --git a/src/xrt/state_trackers/gui/gui_scene_record.c b/src/xrt/state_trackers/gui/gui_scene_record.c new file mode 100644 index 000000000..0d90b32c1 --- /dev/null +++ b/src/xrt/state_trackers/gui/gui_scene_record.c @@ -0,0 +1,415 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Recording scene gui. + * @author Jakob Bornecrantz + * @ingroup gui + */ + + +#include "xrt/xrt_config_have.h" +#include "xrt/xrt_config_drivers.h" + +#include "os/os_threading.h" + +#include "util/u_var.h" +#include "util/u_misc.h" +#include "util/u_sink.h" +#include "util/u_file.h" +#include "util/u_json.h" +#include "util/u_frame.h" +#include "util/u_format.h" + +#include "xrt/xrt_frame.h" +#include "xrt/xrt_prober.h" +#include "xrt/xrt_tracking.h" +#include "xrt/xrt_frameserver.h" + +#ifdef XRT_BUILD_DRIVER_VF +#include "vf/vf_interface.h" +#endif + +#ifdef XRT_BUILD_DRIVER_DEPTHAI +#include "depthai/depthai_interface.h" +#endif + +#include "gui_imgui.h" +#include "gui_common.h" +#include "gui_window_record.h" + +#include "stb_image_write.h" + +#include +#include + + +struct camera_window +{ + struct gui_record_window base; + + struct + { + // Use DepthAI camera. + bool depthai; + + // Use leap_motion. + bool leap_motion; + + // Use index. + bool index; + + // Use ELP. + bool elp; + } use; + + struct + { + struct xrt_frame_context xfctx; + + struct xrt_fs *xfs; + + struct xrt_fs_mode mode; + + char name[256]; + } camera; +}; + +struct record_scene +{ + struct gui_scene base; + + struct camera_window *window; +}; + + + +/* + * + * Camera window functions. + * + */ + +static void +window_set_camera_source(struct camera_window *cw, uint32_t width, uint32_t height, enum xrt_format format) +{ + cw->base.source.width = width; + cw->base.source.height = height; + cw->base.source.format = format; + + // Touch up. + if (cw->use.leap_motion) { + cw->base.source.width = cw->base.source.width * 2; + cw->base.source.format = XRT_FORMAT_L8; + } + + // If it's a large source, scale to 50% + if (cw->base.source.width > 640) { + cw->base.texture.scale = 2; + } +} + +static void +window_destroy(struct camera_window *cw) +{ + // Stop the camera if we have one. + xrt_frame_context_destroy_nodes(&cw->camera.xfctx); + cw->camera.xfs = NULL; + + // Now it's safe to close the window. + gui_window_record_close(&cw->base); + + // And free. + free(cw); +} + +static bool +window_has_source(struct camera_window *cw) +{ + return cw->camera.xfs != NULL; +} + +static struct camera_window * +window_create(struct gui_program *p, const char *camera) +{ + struct camera_window *cw = U_TYPED_CALLOC(struct camera_window); + + // First init recording window. + if (!gui_window_record_init(&cw->base)) { + free(cw); + return NULL; + } + + cw->use.index = camera == NULL ? false : strcmp(camera, "index") == 0; + cw->use.leap_motion = camera == NULL ? false : strcmp(camera, "leap_motion") == 0; + cw->use.depthai = camera == NULL ? false : strcmp(camera, "depthai") == 0; + cw->use.elp = camera == NULL ? false : strcmp(camera, "elp") == 0; + + if (!cw->use.index && !cw->use.leap_motion && !cw->use.depthai && !cw->use.elp) { + U_LOG_W( + "Can't recongnize camera name '%s', options are 'elp', depthai', index' & 'leap_motion'." + "\n\tFalling back to 'index'.", + camera); + cw->use.index = true; + } + + return cw; +} + + +/* + * + * DepthAI functions + * + */ + +#ifdef XRT_BUILD_DRIVER_DEPTHAI +static void +create_depthai(struct camera_window *cw) +{ + // Should we be using a DepthAI camera? + if (!cw->use.depthai) { + return; + } + + cw->camera.xfs = depthai_fs_single_rgb(&cw->camera.xfctx); + + // Just after the camera create a quirk stream. + struct u_sink_quirk_params qp; + U_ZERO(&qp); + qp.stereo_sbs = false; + qp.ps4_cam = false; + qp.leap_motion = false; + + struct xrt_frame_sink *tmp = &cw->base.sink; + u_sink_quirk_create(&cw->camera.xfctx, tmp, &qp, &tmp); + + struct xrt_fs_mode *modes = NULL; + uint32_t mode_count = 0; + xrt_fs_enumerate_modes(cw->camera.xfs, &modes, &mode_count); + assert(mode_count > 0); + + // Just use the first one. + uint32_t mode_index = 0; + + window_set_camera_source( // + cw, // + modes[mode_index].width, // + modes[mode_index].height, // + modes[mode_index].format); // + + free(modes); + modes = NULL; + + // Now that we have setup a node graph, start it. + xrt_fs_stream_start(cw->camera.xfs, tmp, XRT_FS_CAPTURE_TYPE_CALIBRATION, mode_index); +} +#endif /* XRT_BUILD_DRIVER_DEPTHAI */ + + +/* + * + * Video frame functions + * + */ + +#ifdef XRT_BUILD_DRIVER_VF +static void +create_videotestsrc(struct camera_window *cw) +{ + uint32_t width = 1920; + uint32_t height = 960; + cw->camera.xfs = vf_fs_videotestsource(&cw->camera.xfctx, width, height); + + // Just after the camera create a quirk stream. + struct u_sink_quirk_params qp; + U_ZERO(&qp); + qp.stereo_sbs = false; + qp.ps4_cam = false; + qp.leap_motion = false; + + struct xrt_frame_sink *tmp = NULL; + u_sink_quirk_create(&cw->camera.xfctx, &cw->base.sink, &qp, &tmp); + + window_set_camera_source( // + cw, // + width, // + height, // + XRT_FORMAT_R8G8B8); // + + // Now that we have setup a node graph, start it (mode index is hardcoded to 0). + xrt_fs_stream_start(cw->camera.xfs, tmp, XRT_FS_CAPTURE_TYPE_CALIBRATION, 0); +} +#endif /* XRT_BUILD_DRIVER_VF */ + + +/* + * + * Prober functions. + * + */ + + +static bool +is_camera_elp(const char *product, const char *manufacturer) +{ + return strcmp(product, "3D USB Camera") == 0 && strcmp(manufacturer, "3D USB Camera") == 0; +} + +static bool +is_camera_index(const char *product, const char *manufacturer) +{ + return strcmp(product, "3D Camera") == 0 && strcmp(manufacturer, "Etron Technology, Inc.") == 0; +} + +static bool +is_camera_leap_motion(const char *product, const char *manufacturer) +{ + return strcmp(product, "Leap Motion Controller") == 0 && strcmp(manufacturer, "Leap Motion") == 0; +} + +static void +on_video_device(struct xrt_prober *xp, + struct xrt_prober_device *pdev, + const char *product, + const char *manufacturer, + const char *serial, + void *ptr) +{ + struct camera_window *rw = (struct camera_window *)ptr; + + if (rw->camera.xfs != NULL) { + return; + } + + + // Hardcoded for the Index. + if (rw->use.elp && !is_camera_elp(product, manufacturer)) { + return; + } + + // Hardcoded for the Index. + if (rw->use.index && !is_camera_index(product, manufacturer)) { + return; + } + + // Hardcoded for the Leap Motion. + if (rw->use.leap_motion && !is_camera_leap_motion(product, manufacturer)) { + return; + } + + snprintf(rw->camera.name, sizeof(rw->camera.name), "%s-%s", product, serial); + + xrt_prober_open_video_device(xp, pdev, &rw->camera.xfctx, &rw->camera.xfs); + + struct xrt_frame_sink *tmp = &rw->base.sink; + + if (rw->use.leap_motion) { + // De-interleaving. + u_sink_deinterleaver_create(&rw->camera.xfctx, tmp, &tmp); + } + + // Just use the first one. + uint32_t mode_index = 0; + + // Just after the camera create a quirk stream. + struct u_sink_quirk_params qp; + U_ZERO(&qp); + qp.stereo_sbs = false; + qp.ps4_cam = false; + qp.leap_motion = rw->use.leap_motion; + + // Tweaks for the ELP camera. + if (rw->use.elp) { + qp.stereo_sbs = true; + mode_index = 2; + } + + u_sink_quirk_create(&rw->camera.xfctx, tmp, &qp, &tmp); + + struct xrt_fs_mode *modes = NULL; + uint32_t mode_count = 0; + xrt_fs_enumerate_modes(rw->camera.xfs, &modes, &mode_count); + assert(mode_count > 0); + + window_set_camera_source( // + rw, // + modes[mode_index].width, // + modes[mode_index].height, // + modes[mode_index].format); // + + free(modes); + modes = NULL; + + // Now that we have setup a node graph, start it. + xrt_fs_stream_start(rw->camera.xfs, tmp, XRT_FS_CAPTURE_TYPE_CALIBRATION, mode_index); +} + + +/* + * + * Scene functions. + * + */ + +static void +scene_render(struct gui_scene *scene, struct gui_program *p) +{ + static ImVec2 button_dims = {0, 0}; + struct record_scene *rs = (struct record_scene *)scene; + + + igBegin("Record-a-tron!", NULL, 0); + + gui_window_record_render(&rs->window->base, p); + + igSeparator(); + + if (igButton("Exit", button_dims)) { + gui_scene_delete_me(p, &rs->base); + } + + igEnd(); +} + +static void +scene_destroy(struct gui_scene *scene, struct gui_program *p) +{ + struct record_scene *rs = (struct record_scene *)scene; + + if (rs->window != NULL) { + window_destroy(rs->window); + rs->window = NULL; + } + + free(rs); +} + +void +gui_scene_record(struct gui_program *p, const char *camera) +{ + struct record_scene *rs = U_TYPED_CALLOC(struct record_scene); + + rs->base.render = scene_render; + rs->base.destroy = scene_destroy; + + rs->window = window_create(p, camera); + +#ifdef XRT_BUILD_DRIVER_DEPTHAI + if (!window_has_source(rs->window)) { + create_depthai(rs->window); + } +#endif + + if (!window_has_source(rs->window)) { + xrt_prober_list_video_devices(p->xp, on_video_device, rs->window); + } + +#ifdef XRT_BUILD_DRIVER_VF + if (!window_has_source(rs->window)) { + create_videotestsrc(rs->window); + } +#endif + + gui_scene_push_front(p, &rs->base); +} diff --git a/src/xrt/state_trackers/gui/gui_scene_tracking_overrides.c b/src/xrt/state_trackers/gui/gui_scene_tracking_overrides.c new file mode 100644 index 000000000..abd7a0766 --- /dev/null +++ b/src/xrt/state_trackers/gui/gui_scene_tracking_overrides.c @@ -0,0 +1,367 @@ +// Copyright 2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief A very small scene that lets the user configure tracking overrides. + * @author Christoph Haag + * @author Jakob Bornecrantz + * @ingroup gui + */ + +#include "util/u_misc.h" +#include "util/u_format.h" + +#include "util/u_config_json.h" + +#include "math/m_api.h" + +#include "xrt/xrt_prober.h" +#include "xrt/xrt_settings.h" + +#include "gui_common.h" +#include "gui_imgui.h" + +#include "bindings/b_generated_bindings.h" + +struct gui_tracking_overrides +{ + struct gui_scene base; + + int editing_override; + + bool add_one; + int add_target; + int add_tracker; + + struct u_config_json config; + + struct xrt_pose reset_offset; + + size_t num_overrides; + struct xrt_tracking_override overrides[XRT_MAX_TRACKING_OVERRIDES]; +}; + +static char *override_type_str[2] = { + [XRT_TRACKING_OVERRIDE_DIRECT] = "direct", + [XRT_TRACKING_OVERRIDE_ATTACHED] = "attached", +}; + +static ImVec2 button_dims = {256 + 64, 0}; + +/* + * + * Internal functions. + * + */ + +#define NAME_LENGTH XRT_DEVICE_NAME_LEN * 2 + 5 + +static void +make_name(struct xrt_device *xdev, char *buf) +{ + snprintf(buf, NAME_LENGTH, "%s | %s", xdev->str, xdev->serial); +} + +static void +handle_draggable_vec3_f32(const char *name, struct xrt_vec3 *v, const struct xrt_vec3 *reset) +{ + float min = -256.0f; + float max = 256.0f; + char tmp[256]; + + snprintf(tmp, sizeof(tmp), "%s.reset", name); + + if (igArrowButton(tmp, ImGuiDir_Left)) { + *v = *reset; + } + + igSameLine(0, 3); + igDragFloat3(name, (float *)v, 0.005f, min, max, "%+f", 1.0f); +} + +static void +handle_draggable_quat(const char *name, struct xrt_quat *q, const struct xrt_quat *reset) +{ + float min = -1.0f; + float max = 1.0f; + + char tmp[256]; + + snprintf(tmp, sizeof(tmp), "%s.reset", name); + + if (igArrowButton(tmp, ImGuiDir_Left)) { + *q = *reset; + } + + igSameLine(0, 3); + igDragFloat4(name, (float *)q, 0.005f, min, max, "%+f", 1.0f); + + // Avoid invalid + if (q->x == 0.0f && q->y == 0.0f && q->z == 0.0f && q->w == 0.0f) { + q->w = 1.0f; + } + + // And make sure it's a unit rotation. + math_quat_normalize(q); +} + +static bool +get_indices(struct gui_program *p, + struct gui_tracking_overrides *ts, + struct xrt_tracking_override *override, + int *out_target, + int *out_tracker) +{ + bool has_target = false; + bool has_tracker = false; + + for (int i = 0; i < NUM_XDEVS; i++) { + if (!p->xdevs[i]) { + continue; + } + + if (strcmp(p->xdevs[i]->serial, override->target_device_serial) == 0) { + has_target = true; + *out_target = i; + } + + if (strcmp(p->xdevs[i]->serial, override->tracker_device_serial) == 0) { + has_tracker = true; + *out_tracker = i; + } + } + + return has_tracker && has_target; +} + +static struct xrt_pose identity = {.position = {.x = 0, .y = 0, .z = 0}, + .orientation = {.x = 0, .y = 0, .z = 0, .w = 1}}; + +static void +add_one(struct gui_program *p, struct gui_tracking_overrides *ts) +{ + igBegin("Target Device", NULL, 0); + for (int i = 0; i < 8; i++) { + if (!p->xdevs[i]) { + continue; + } + + char buf[NAME_LENGTH]; + make_name(p->xdevs[i], buf); + + bool selected = ts->add_target == i; + if (igCheckbox(buf, &selected)) { + ts->add_target = i; + } + } + igEnd(); + + igBegin("Tracker Device", NULL, 0); + for (int i = 0; i < 8; i++) { + if (!p->xdevs[i]) { + continue; + } + + char buf[NAME_LENGTH]; + make_name(p->xdevs[i], buf); + + bool selected = ts->add_tracker == i; + if (igCheckbox(buf, &selected)) { + ts->add_tracker = i; + } + } + igEnd(); + + if (ts->add_target >= 0 && ts->add_tracker >= 0 && ts->add_target != ts->add_tracker) { + struct xrt_tracking_override *o = &ts->overrides[ts->num_overrides]; + strncpy(o->target_device_serial, p->xdevs[ts->add_target]->serial, XRT_DEVICE_NAME_LEN); + strncpy(o->tracker_device_serial, p->xdevs[ts->add_tracker]->serial, XRT_DEVICE_NAME_LEN); + o->offset = identity; + + // set input_name to the first pose in the inputs + for (uint32_t i = 0; i < p->xdevs[ts->add_tracker]->num_inputs; i++) { + enum xrt_input_name input_name = p->xdevs[ts->add_tracker]->inputs[i].name; + if (XRT_GET_INPUT_TYPE(input_name) != XRT_INPUT_TYPE_POSE) { + continue; + } + o->input_name = input_name; + break; + } + + ts->num_overrides += 1; + + ts->add_target = -1; + ts->add_tracker = -1; + + ts->add_one = false; + + // immediately open for editing + ts->editing_override = ts->num_overrides - 1; + } +} + +static void +scene_render(struct gui_scene *scene, struct gui_program *p) +{ + struct gui_tracking_overrides *ts = (struct gui_tracking_overrides *)scene; + + // don't edit and add at the same time + if (ts->add_one) { + ts->editing_override = -1; + } + + if (ts->editing_override >= 0) { + struct xrt_tracking_override *o = &ts->overrides[ts->editing_override]; + + igBegin("Tracker Device Offset", NULL, 0); + int target = -1, tracker = -1; + if (get_indices(p, ts, o, &target, &tracker)) { + igText("Editing %s [%s] <- %s [%s]", p->xdevs[target]->str, o->target_device_serial, + p->xdevs[tracker]->str, o->tracker_device_serial); + } else { + igText("Editing unconnected %s <- %s", o->target_device_serial, o->tracker_device_serial); + } + handle_draggable_vec3_f32("Position", &o->offset.position, &ts->reset_offset.position); + handle_draggable_quat("Orientation", &o->offset.orientation, &ts->reset_offset.orientation); + + igText("Tracking Override Type"); + for (int i = 0; i < 2; i++) { + bool selected = (int)o->override_type == i; + if (igCheckbox(override_type_str[i], &selected)) { + o->override_type = (enum xrt_tracking_override_type)i; + } + } + + if (tracker >= 0) { + igText("Tracker Input Pose Name"); + for (uint32_t i = 0; i < p->xdevs[tracker]->num_inputs; i++) { + enum xrt_input_name input_name = p->xdevs[tracker]->inputs[i].name; + if (XRT_GET_INPUT_TYPE(input_name) != XRT_INPUT_TYPE_POSE) { + continue; + } + + const char *name_str = xrt_input_name_string(input_name); + bool selected = o->input_name == input_name; + if (igCheckbox(name_str, &selected)) { + o->input_name = input_name; + } + } + igEnd(); + } + } + + if (ts->add_one) { + add_one(p, ts); + } + + igBegin("Tracking Overrides", NULL, 0); + + igText("Existing Overrides"); + for (size_t i = 0; i < ts->num_overrides; i++) { + // make the delete buttons work + igPushIDInt(i); + + igSeparator(); + + bool checked = ts->editing_override == (int)i; + + char buf[XRT_DEVICE_NAME_LEN * 2 + 10]; + snprintf(buf, sizeof(buf), "%s <- %s", ts->overrides[i].target_device_serial, + ts->overrides[i].tracker_device_serial); + if (igCheckbox(buf, &checked)) { + + // abort adding override when clicking to edit one + ts->add_one = false; + + ts->editing_override = i; + ts->reset_offset = ts->overrides[i].offset; + } + if (igButton("Delete this one", button_dims)) { + for (size_t j = i; j < ts->num_overrides - 1; j++) { + ts->overrides[j] = ts->overrides[j + 1]; + } + ts->num_overrides--; + if (ts->editing_override >= (int)i) { + ts->editing_override -= 1; + } + } + + igSeparator(); + + igPopID(); + } + + igSeparator(); + + if (igButton("Add one", button_dims)) { + if (ts->num_overrides < XRT_MAX_TRACKING_OVERRIDES) { + ts->add_one = true; + } + } + + igSeparator(); + + if (igButton("Save", button_dims)) { + u_config_json_save_overrides(&ts->config, ts->overrides, ts->num_overrides); + u_config_json_close(&ts->config); + gui_scene_delete_me(p, scene); + } + + if (igButton("Exit", button_dims)) { + gui_scene_delete_me(p, scene); + } + + igEnd(); +} + +static void +scene_destroy(struct gui_scene *scene, struct gui_program *p) +{ + // struct tracking_overrides *ts = (struct tracking_overrides *)scene; + + free(scene); +} + +static struct gui_tracking_overrides * +create(struct gui_program *p) +{ + struct gui_tracking_overrides *ts = U_TYPED_CALLOC(struct gui_tracking_overrides); + + ts->base.render = scene_render; + ts->base.destroy = scene_destroy; + + ts->editing_override = -1; + ts->add_one = false; + ts->add_target = -1; + ts->add_tracker = -1; + + u_config_json_open_or_create_main_file(&ts->config); + u_config_json_get_tracking_overrides(&ts->config, ts->overrides, &ts->num_overrides); + + return ts; +} + + +/* + * + * 'Exported' functions. + * + */ + +void +gui_scene_tracking_overrides(struct gui_program *p) +{ + if (p->xp == NULL) { + // No prober, nothing to create. + return; + } + + // If we have created a prober select devices now. + if (p->xp != NULL) { + gui_prober_select(p); + } + + struct gui_tracking_overrides *ts = create(p); + + gui_scene_push_front(p, &ts->base); +} diff --git a/src/xrt/state_trackers/gui/gui_scene_video.c b/src/xrt/state_trackers/gui/gui_scene_video.c index 8062276b4..41aa2f313 100644 --- a/src/xrt/state_trackers/gui/gui_scene_video.c +++ b/src/xrt/state_trackers/gui/gui_scene_video.c @@ -7,16 +7,22 @@ * @ingroup gui */ -#include "util/u_misc.h" -#include "util/u_format.h" - #include "xrt/xrt_prober.h" #include "xrt/xrt_settings.h" #include "xrt/xrt_frameserver.h" +#include "xrt/xrt_config_drivers.h" + +#include "util/u_misc.h" +#include "util/u_format.h" +#include "util/u_logging.h" #include "gui_common.h" #include "gui_imgui.h" +#ifdef XRT_BUILD_DRIVER_DEPTHAI +#include "depthai/depthai_interface.h" +#endif + /*! * A GUI scene that lets the user select a user device. @@ -44,6 +50,24 @@ static ImVec2 button_dims = {256 + 64, 0}; * */ +#ifdef XRT_BUILD_DRIVER_DEPTHAI +static void +create_depthai(struct video_select *vs) +{ + vs->xfctx = U_TYPED_CALLOC(struct xrt_frame_context); + + vs->xfs = depthai_fs_single_rgb(vs->xfctx); + if (vs->xfs == NULL) { + U_LOG_E("Failed to open DepthAI camera!"); + free(vs->xfctx); + vs->xfctx = NULL; + return; + } + + xrt_fs_enumerate_modes(vs->xfs, &vs->modes, &vs->num_modes); +} +#endif /* XRT_BUILD_DRIVER_DEPTHAI */ + static void on_video_device(struct xrt_prober *xp, struct xrt_prober_device *pdev, @@ -59,7 +83,9 @@ on_video_device(struct xrt_prober *xp, } char buf[256] = {0}; - snprintf(buf, sizeof(buf), "%04x:%04x '%s' '%s'\n", pdev->vendor_id, pdev->product_id, product, serial); + uint16_t vendor_id = pdev ? pdev->vendor_id : -1; + uint16_t product_id = pdev ? pdev->product_id : -1; + snprintf(buf, sizeof(buf), "%04x:%04x '%s' '%s'\n", vendor_id, product_id, product, serial); if (!igButton(buf, button_dims)) { return; } @@ -67,7 +93,15 @@ on_video_device(struct xrt_prober *xp, snprintf(vs->settings->camera_name, sizeof(vs->settings->camera_name), "%s", product); vs->xfctx = U_TYPED_CALLOC(struct xrt_frame_context); + xrt_prober_open_video_device(xp, pdev, vs->xfctx, &vs->xfs); + if (vs->xfs == NULL) { + U_LOG_E("Failed to open camera!"); + free(vs->xfctx); + vs->xfctx = NULL; + return; + } + xrt_fs_enumerate_modes(vs->xfs, &vs->modes, &vs->num_modes); } @@ -91,6 +125,13 @@ scene_render(struct gui_scene *scene, struct gui_program *p) // If we have not found any modes keep showing the devices. if (vs->xfs == NULL) { xrt_prober_list_video_devices(p->xp, on_video_device, vs); + +#ifdef XRT_BUILD_DRIVER_DEPTHAI + igSeparator(); + if (igButton("DepthAI", button_dims)) { + create_depthai(vs); + } +#endif } else if (vs->num_modes == 0) { // No modes on it :( igText("No modes found on '%s'!", vs->xfs->name); diff --git a/src/xrt/state_trackers/gui/gui_stb.c b/src/xrt/state_trackers/gui/gui_stb.c new file mode 100644 index 000000000..fc7a0c7a3 --- /dev/null +++ b/src/xrt/state_trackers/gui/gui_stb.c @@ -0,0 +1,10 @@ +// Copyright 2020, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Compile the STB image write implementation. + * @author Jakob Bornecrantz + */ + +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include "stb_image_write.h" diff --git a/src/xrt/state_trackers/gui/gui_window_record.c b/src/xrt/state_trackers/gui/gui_window_record.c new file mode 100644 index 000000000..b41bbd0b0 --- /dev/null +++ b/src/xrt/state_trackers/gui/gui_window_record.c @@ -0,0 +1,335 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Recording window gui. + * @author Jakob Bornecrantz + * @ingroup gui + */ + +#include "xrt/xrt_config_have.h" +#include "xrt/xrt_config_drivers.h" + +#include "os/os_threading.h" + +#include "util/u_var.h" +#include "util/u_misc.h" +#include "util/u_sink.h" +#include "util/u_file.h" +#include "util/u_json.h" +#include "util/u_frame.h" +#include "util/u_format.h" + +#include "xrt/xrt_frame.h" +#include "xrt/xrt_prober.h" +#include "xrt/xrt_tracking.h" +#include "xrt/xrt_frameserver.h" + +#include "gui_window_record.h" + +#ifdef XRT_HAVE_GST +#include "gstreamer/gst_sink.h" +#include "gstreamer/gst_pipeline.h" +#endif + +#include "gui_imgui.h" +#include "gui_common.h" +#include "gui_window_record.h" + +#include "stb_image_write.h" + +#include +#include + + +/* + * + * GStreamer functions. + * + */ + +#ifdef XRT_HAVE_GST +static void +create_pipeline(struct gui_record_window *rw) +{ + const char *source_name = "source_name"; + const char *bitrate = NULL; + const char *speed_preset = NULL; + + char pipeline_string[2048]; + + switch (rw->gst.bitrate) { + default: + case GUI_RECORD_BITRATE_4096: bitrate = "4096"; break; + case GUI_RECORD_BITRATE_2048: bitrate = "2048"; break; + case GUI_RECORD_BITRATE_1024: bitrate = "1024"; break; + } + + switch (rw->gst.pipeline) { + case GUI_RECORD_PIPELINE_SOFTWARE_FAST: speed_preset = "fast"; break; + case GUI_RECORD_PIPELINE_SOFTWARE_MEDIUM: speed_preset = "medium"; break; + case GUI_RECORD_PIPELINE_SOFTWARE_SLOW: speed_preset = "slow"; break; + case GUI_RECORD_PIPELINE_SOFTWARE_VERYSLOW: speed_preset = "veryslow"; break; + default: break; + } + + if (speed_preset != NULL) { + snprintf(pipeline_string, // + sizeof(pipeline_string), // + "appsrc name=\"%s\" ! " + "queue ! " + "videoconvert ! " + "queue ! " + "x264enc bitrate=\"%s\" speed-preset=\"%s\" ! " + "h264parse ! " + "queue ! " + "mp4mux ! " + "filesink location=\"%s\"", + source_name, bitrate, speed_preset, rw->gst.filename); + } else { + snprintf(pipeline_string, // + sizeof(pipeline_string), // + "appsrc name=\"%s\" ! " + "queue ! " + "videoconvert ! " + "video/x-raw,format=NV12 ! " + "queue ! " + "vaapih264enc rate-control=cbr bitrate=\"%s\" tune=high-compression ! " + "video/x-h264,profile=main ! " + "h264parse ! " + "queue ! " + "mp4mux ! " + "filesink location=\"%s\"", + source_name, bitrate, rw->gst.filename); + } + + struct xrt_frame_sink *tmp = NULL; + struct gstreamer_pipeline *gp = NULL; + + gstreamer_pipeline_create_from_string(&rw->gst.xfctx, pipeline_string, &gp); + + uint32_t width = rw->source.width; + uint32_t height = rw->source.height; + enum xrt_format format = rw->source.format; + + bool do_convert = false; + if (format == XRT_FORMAT_MJPEG) { + format = XRT_FORMAT_R8G8B8; + do_convert = true; + } + + struct gstreamer_sink *gs = NULL; + gstreamer_sink_create_with_pipeline(gp, width, height, format, source_name, &gs, &tmp); + if (do_convert) { + u_sink_create_to_r8g8b8_or_l8(&rw->gst.xfctx, tmp, &tmp); + } + u_sink_queue_create(&rw->gst.xfctx, tmp, &tmp); + + os_mutex_lock(&rw->gst.mutex); + rw->gst.gs = gs; + rw->gst.sink = tmp; + rw->gst.gp = gp; + gstreamer_pipeline_play(rw->gst.gp); + os_mutex_unlock(&rw->gst.mutex); +} + +static void +destroy_pipeline(struct gui_record_window *rw) +{ + U_LOG_D("Called"); + + // Make sure we are not streaming any more frames into the pipeline. + os_mutex_lock(&rw->gst.mutex); + rw->gst.gs = NULL; + rw->gst.sink = NULL; + os_mutex_unlock(&rw->gst.mutex); + + // Stop the pipeline. + gstreamer_pipeline_stop(rw->gst.gp); + rw->gst.gp = NULL; + + xrt_frame_context_destroy_nodes(&rw->gst.xfctx); +} + +static void +draw_gst(struct gui_record_window *rw) +{ + static ImVec2 button_dims = {0, 0}; + + if (!igCollapsingHeaderBoolPtr("Record", NULL, ImGuiTreeNodeFlags_DefaultOpen)) { + return; + } + + + os_mutex_lock(&rw->gst.mutex); + bool recording = rw->gst.gp != NULL; + os_mutex_unlock(&rw->gst.mutex); + + igComboStr("Pipeline", (int *)&rw->gst.pipeline, "SW Fast\0SW Medium\0SW Slow\0SW Veryslow\0VAAPI H264\0\0", 5); + igComboStr("Bitrate", (int *)&rw->gst.bitrate, "4096bps\0002048bps\0001024bps\0\0", 3); + + igInputText("Filename", rw->gst.filename, sizeof(rw->gst.filename), 0, NULL, NULL); + + if (!recording && igButton("Start", button_dims)) { + create_pipeline(rw); + } + + if (recording && igButton("Stop", button_dims)) { + destroy_pipeline(rw); + } +} +#endif + + +/* + * + * Misc helpers and interface functions. + * + */ + +static void +window_draw_misc(struct gui_record_window *rw) +{ + if (!igCollapsingHeaderBoolPtr("Misc", NULL, ImGuiTreeNodeFlags_DefaultOpen)) { + return; + } + + static ImVec2 button_dims = {0, 0}; + bool plus = igButton("+", button_dims); + igSameLine(0.0f, 4.0f); + bool minus = igButton("-", button_dims); + igSameLine(0.0f, 4.0f); + + if (rw->texture.scale == 1) { + igText("Scale 100%%"); + } else { + igText("Scale 1/%i", rw->texture.scale); + } + + if (plus && rw->texture.scale > 1) { + rw->texture.scale--; + } + if (minus && rw->texture.scale < 6) { + rw->texture.scale++; + } + + igText("Sequence %u", (uint32_t)rw->texture.ogl->seq); +} + +static void +window_frame(struct xrt_frame_sink *xfs, struct xrt_frame *xf) +{ + struct gui_record_window *rw = container_of(xfs, struct gui_record_window, sink); + + if (rw->source.width != xf->width || rw->source.height != xf->height || rw->source.format != xf->format) { + assert(rw->source.width == 0 && rw->source.height == 0); + + rw->source.width = xf->width; + rw->source.height = xf->height; + rw->source.format = xf->format; + } + +#ifdef XRT_HAVE_GST + os_mutex_lock(&rw->gst.mutex); + if (rw->gst.sink != NULL) { + xrt_sink_push_frame(rw->gst.sink, xf); + } + os_mutex_unlock(&rw->gst.mutex); +#endif + + xrt_sink_push_frame(rw->texture.sink, xf); +} + + +/* + * + * 'Exported' functions. + * + */ + +bool +gui_window_record_init(struct gui_record_window *rw) +{ + // Basic init. + rw->sink.push_frame = window_frame; + + // Mutex first. +#ifdef XRT_HAVE_GST + int ret = os_mutex_init(&rw->gst.mutex); + if (ret < 0) { + return false; + } + + snprintf(rw->gst.filename, sizeof(rw->gst.filename), "/tmp/capture.mp4"); +#endif + + // Setup the preview texture. + rw->texture.scale = 1; + struct xrt_frame_sink *tmp = NULL; + rw->texture.ogl = gui_ogl_sink_create("View", &rw->texture.xfctx, &tmp); + u_sink_create_to_r8g8b8_or_l8(&rw->texture.xfctx, tmp, &tmp); + u_sink_queue_create(&rw->texture.xfctx, tmp, &rw->texture.sink); + + return true; +} + +void +gui_window_record_render(struct gui_record_window *rw, struct gui_program *p) +{ + // Make all IDs unique. + igPushIDPtr(rw); + + gui_ogl_sink_update(rw->texture.ogl); + + struct gui_ogl_texture *tex = rw->texture.ogl; + + int w = tex->w / rw->texture.scale; + int h = tex->h / rw->texture.scale; + + ImVec2 size = {(float)w, (float)h}; + ImVec2 uv0 = {0, 0}; + ImVec2 uv1 = {1, 1}; + ImVec4 white = {1, 1, 1, 1}; + ImTextureID id = (ImTextureID)(intptr_t)tex->id; + igImage(id, size, uv0, uv1, white, white); + +#ifdef XRT_HAVE_GST + draw_gst(rw); +#endif + + window_draw_misc(rw); + + // Pop the ID making everything unique. + igPopID(); +} + +void +gui_window_record_close(struct gui_record_window *rw) +{ + // Stop and remove the recording pipeline first. +#ifdef XRT_HAVE_GST + if (rw->gst.gp != NULL) { + os_mutex_lock(&rw->gst.mutex); + rw->gst.gs = NULL; + rw->gst.sink = NULL; + os_mutex_unlock(&rw->gst.mutex); + + gstreamer_pipeline_stop(rw->gst.gp); + rw->gst.gp = NULL; + xrt_frame_context_destroy_nodes(&rw->gst.xfctx); + } +#endif + + xrt_frame_context_destroy_nodes(&rw->texture.xfctx); + + /* + * This is safe to do, because we require that our sink 'window_frame' + * function is not called when close is called. + */ + rw->texture.sink = NULL; + rw->texture.ogl = NULL; + +#ifdef XRT_HAVE_GST + os_mutex_destroy(&rw->gst.mutex); +#endif +} diff --git a/src/xrt/state_trackers/gui/gui_window_record.h b/src/xrt/state_trackers/gui/gui_window_record.h new file mode 100644 index 000000000..73a908cf7 --- /dev/null +++ b/src/xrt/state_trackers/gui/gui_window_record.h @@ -0,0 +1,114 @@ +// Copyright 2019-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Recording window gui. + * @author Jakob Bornecrantz + * @ingroup gui + */ + +#pragma once + +#include "xrt/xrt_frame.h" +#include "xrt/xrt_defines.h" +#include "xrt/xrt_config_have.h" + + +#ifdef __cplusplus +extern "C" { +#endif + +struct xrt_frame_sink; +struct gstreamer_sink; +struct gstreamer_pipeline; +struct gui_program; +struct gui_ogl_texture; + + +enum gui_record_bitrate +{ + GUI_RECORD_BITRATE_4096, + GUI_RECORD_BITRATE_2048, + GUI_RECORD_BITRATE_1024, +}; + +enum gui_record_pipeline +{ + GUI_RECORD_PIPELINE_SOFTWARE_FAST, + GUI_RECORD_PIPELINE_SOFTWARE_MEDIUM, + GUI_RECORD_PIPELINE_SOFTWARE_SLOW, + GUI_RECORD_PIPELINE_SOFTWARE_VERYSLOW, + GUI_RECORD_PIPELINE_VAAPI_H246, +}; + +struct gui_record_window +{ + struct xrt_frame_sink sink; + + struct + { + uint32_t width, height; + enum xrt_format format; + } source; + + struct + { + struct xrt_frame_context xfctx; + + int scale; + + struct xrt_frame_sink *sink; + struct gui_ogl_texture *ogl; + } texture; + +#ifdef XRT_HAVE_GST + struct + { + enum gui_record_bitrate bitrate; + + enum gui_record_pipeline pipeline; + + struct xrt_frame_context xfctx; + + //! When not null we are recording. + struct xrt_frame_sink *sink; + + //! Protects sink + struct os_mutex mutex; + + //! App sink we are pushing frames into. + struct gstreamer_sink *gs; + + //! Recording pipeline. + struct gstreamer_pipeline *gp; + + char filename[512]; + } gst; +#endif +}; + + +/*! + * Initialise a embeddable record window. + */ +bool +gui_window_record_init(struct gui_record_window *rw); + +/*! + * Renders all controls of a record window. + */ +void +gui_window_record_render(struct gui_record_window *rw, struct gui_program *p); + +/*! + * Frees all resources assocciated with a record window. Make sure to only call + * this function on the main gui thread, and that nothing is pushing into the + * record windows sink. + */ +void +gui_window_record_close(struct gui_record_window *rw); + + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/state_trackers/gui/meson.build b/src/xrt/state_trackers/gui/meson.build index 65cb8367a..2f6807cd0 100644 --- a/src/xrt/state_trackers/gui/meson.build +++ b/src/xrt/state_trackers/gui/meson.build @@ -1,4 +1,4 @@ -# Copyright 2019-2020, Collabora, Ltd. +# Copyright 2019-2021, Collabora, Ltd. # SPDX-License-Identifier: BSL-1.0 gui_sources = [ @@ -10,8 +10,13 @@ gui_sources = [ 'gui_scene_calibrate.c', 'gui_scene_debug.c', 'gui_scene_main_menu.c', + 'gui_scene_record.c', 'gui_scene_remote.c', 'gui_scene_video.c', + 'gui_scene_tracking_overrides.c', + 'gui_stb.c', + 'gui_window_record.c', + 'gui_window_record.h', '../../../external/imgui/imgui/cimgui.cpp', '../../../external/imgui/imgui/cimgui.h', '../../../external/imgui/imgui/cimplot.cpp', @@ -27,6 +32,7 @@ gui_sources = [ '../../../external/imgui/imgui/imgui_widgets.cpp', '../../../external/imgui/imgui/implot.cpp', '../../../external/imgui/imgui/implot.h', + '../../../external/imgui/imgui/implot_demo.cpp', '../../../external/imgui/imgui/implot_internal.h', '../../../external/imgui/imgui/implot_items.cpp', '../../../external/imgui/imgui/imstb_rectpack.h', @@ -38,6 +44,14 @@ gui_sources = [ gui_deps = [aux, xrt_config_have] +if 'vf' in drivers + gui_deps += [drv_vf] +endif + +if 'depthai' in drivers + gui_deps += [drv_depthai] +endif + lib_st_gui = static_library( 'st_gui', files(gui_sources), @@ -47,6 +61,7 @@ lib_st_gui = static_library( glad_include, cjson_include, imgui_include, + stb_include, ], dependencies: gui_deps, ) diff --git a/src/xrt/state_trackers/oxr/CMakeLists.txt b/src/xrt/state_trackers/oxr/CMakeLists.txt index 13accff35..aaa3c34b7 100644 --- a/src/xrt/state_trackers/oxr/CMakeLists.txt +++ b/src/xrt/state_trackers/oxr/CMakeLists.txt @@ -29,6 +29,7 @@ set(OXR_SOURCE_FILES oxr_objects.h oxr_path.c oxr_session.c + oxr_session_frame_end.c oxr_space.c oxr_swapchain.c oxr_system.c @@ -39,7 +40,7 @@ set(OXR_SOURCE_FILES if(XRT_HAVE_VULKAN) list(APPEND OXR_SOURCE_FILES - oxr_session_vk.c + oxr_session_gfx_vk.c oxr_swapchain_vk.c oxr_vulkan.c ) @@ -54,7 +55,7 @@ endif() if(XRT_HAVE_OPENGL OR XRT_HAVE_OPENGLES) list(APPEND OXR_SOURCE_FILES - oxr_session_gl.c + oxr_session_gfx_gl.c oxr_swapchain_gl.c ) endif() @@ -65,14 +66,16 @@ endif() if(XRT_HAVE_EGL) add_definitions(-DXR_USE_PLATFORM_EGL) - list(APPEND OXR_SOURCE_FILES oxr_session_egl.c) + list(APPEND OXR_SOURCE_FILES + oxr_session_gfx_egl.c + ) endif() if(ANDROID) add_definitions(-DXR_USE_PLATFORM_ANDROID) list(APPEND OXR_SOURCE_FILES - oxr_session_gles_android.c + oxr_session_gfx_gles_android.c ) endif() @@ -87,13 +90,12 @@ target_link_libraries(st_oxr PRIVATE aux-includes PUBLIC aux_os - Vulkan::Vulkan ) if(XRT_HAVE_VULKAN) - target_link_libraries(st_oxr PRIVATE Vulkan::Vulkan) + target_link_libraries(st_oxr PUBLIC Vulkan::Vulkan) endif() if(XRT_HAVE_OPENGL OR XRT_HAVE_OPENGLES) - target_link_libraries(st_oxr PRIVATE aux_ogl) + target_link_libraries(st_oxr PUBLIC aux_ogl) endif() if(ANDROID) target_link_libraries(st_oxr PRIVATE aux_android) diff --git a/src/xrt/state_trackers/oxr/meson.build b/src/xrt/state_trackers/oxr/meson.build index 3e267c343..192a2b21a 100644 --- a/src/xrt/state_trackers/oxr/meson.build +++ b/src/xrt/state_trackers/oxr/meson.build @@ -48,9 +48,10 @@ lib_st_oxr = static_library( 'oxr_objects.h', 'oxr_path.c', 'oxr_session.c', - 'oxr_session_gl.c', - 'oxr_session_egl.c', - 'oxr_session_vk.c', + 'oxr_session_frame_end.c', + 'oxr_session_gfx_gl.c', + 'oxr_session_gfx_egl.c', + 'oxr_session_gfx_vk.c', 'oxr_space.c', 'oxr_swapchain.c', 'oxr_swapchain_gl.c', diff --git a/src/xrt/state_trackers/oxr/oxr_api_action.c b/src/xrt/state_trackers/oxr/oxr_api_action.c index 6eff16dcd..e6afdbed0 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_action.c +++ b/src/xrt/state_trackers/oxr/oxr_api_action.c @@ -12,6 +12,7 @@ #include "oxr_handle.h" #include "util/u_debug.h" +#include "util/u_trace_marker.h" #include "oxr_api_funcs.h" #include "oxr_api_verify.h" @@ -31,6 +32,8 @@ XrResult oxr_xrSyncActions(XrSession session, const XrActionsSyncInfo *syncInfo) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrSyncActions"); @@ -57,6 +60,8 @@ oxr_xrSyncActions(XrSession session, const XrActionsSyncInfo *syncInfo) XrResult oxr_xrAttachSessionActionSets(XrSession session, const XrSessionActionSetsAttachInfo *bindInfo) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrAttachSessionActionSets"); @@ -86,6 +91,8 @@ XrResult oxr_xrSuggestInteractionProfileBindings(XrInstance instance, const XrInteractionProfileSuggestedBinding *suggestedBindings) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrSuggestInteractionProfileBindings"); @@ -129,6 +136,8 @@ oxr_xrSuggestInteractionProfileBindings(XrInstance instance, func = oxr_verify_valve_index_controller_subpath; } else if (ip == inst->path_cache.mndx_ball_on_a_stick_controller) { func = oxr_verify_mndx_ball_on_a_stick_controller_subpath; + } else if (ip == inst->path_cache.msft_hand_interaction) { + func = oxr_verify_microsoft_hand_interaction_subpath; } else { return oxr_error(&log, XR_ERROR_PATH_UNSUPPORTED, "(suggestedBindings->interactionProfile == \"%s\") is not " @@ -174,6 +183,8 @@ oxr_xrGetCurrentInteractionProfile(XrSession session, XrPath topLevelUserPath, XrInteractionProfileState *interactionProfile) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst = NULL; struct oxr_session *sess = NULL; struct oxr_logger log; @@ -224,6 +235,8 @@ oxr_xrGetInputSourceLocalizedName(XrSession session, uint32_t *bufferCountOutput, char *buffer) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst = NULL; struct oxr_session *sess; struct oxr_logger log; @@ -276,6 +289,8 @@ oxr_xrGetInputSourceLocalizedName(XrSession session, XrResult oxr_xrCreateActionSet(XrInstance instance, const XrActionSetCreateInfo *createInfo, XrActionSet *actionSet) { + OXR_TRACE_MARKER(); + struct oxr_action_set *act_set = NULL; struct oxr_instance *inst = NULL; struct u_hashset_item *d = NULL; @@ -325,6 +340,8 @@ oxr_xrCreateActionSet(XrInstance instance, const XrActionSetCreateInfo *createIn XrResult oxr_xrDestroyActionSet(XrActionSet actionSet) { + OXR_TRACE_MARKER(); + struct oxr_action_set *act_set; struct oxr_logger log; OXR_VERIFY_ACTIONSET_AND_INIT_LOG(&log, actionSet, act_set, "xrDestroyActionSet"); @@ -342,6 +359,8 @@ oxr_xrDestroyActionSet(XrActionSet actionSet) XrResult oxr_xrCreateAction(XrActionSet actionSet, const XrActionCreateInfo *createInfo, XrAction *action) { + OXR_TRACE_MARKER(); + struct oxr_action_set *act_set; struct u_hashset_item *d = NULL; struct oxr_action *act = NULL; @@ -405,6 +424,8 @@ oxr_xrCreateAction(XrActionSet actionSet, const XrActionCreateInfo *createInfo, XrResult oxr_xrDestroyAction(XrAction action) { + OXR_TRACE_MARKER(); + struct oxr_action *act; struct oxr_logger log; OXR_VERIFY_ACTION_AND_INIT_LOG(&log, action, act, "xrDestroyAction"); @@ -415,6 +436,8 @@ oxr_xrDestroyAction(XrAction action) XrResult oxr_xrGetActionStateBoolean(XrSession session, const XrActionStateGetInfo *getInfo, XrActionStateBoolean *data) { + OXR_TRACE_MARKER(); + struct oxr_session *sess = NULL; struct oxr_action *act = NULL; struct oxr_subaction_paths subaction_paths = {0}; @@ -441,6 +464,8 @@ oxr_xrGetActionStateBoolean(XrSession session, const XrActionStateGetInfo *getIn XrResult oxr_xrGetActionStateFloat(XrSession session, const XrActionStateGetInfo *getInfo, XrActionStateFloat *data) { + OXR_TRACE_MARKER(); + struct oxr_session *sess = NULL; struct oxr_action *act = NULL; struct oxr_subaction_paths subaction_paths = {0}; @@ -467,6 +492,8 @@ oxr_xrGetActionStateFloat(XrSession session, const XrActionStateGetInfo *getInfo XrResult oxr_xrGetActionStateVector2f(XrSession session, const XrActionStateGetInfo *getInfo, XrActionStateVector2f *data) { + OXR_TRACE_MARKER(); + struct oxr_session *sess = NULL; struct oxr_action *act = NULL; struct oxr_subaction_paths subaction_paths = {0}; @@ -493,6 +520,8 @@ oxr_xrGetActionStateVector2f(XrSession session, const XrActionStateGetInfo *getI XrResult oxr_xrGetActionStatePose(XrSession session, const XrActionStateGetInfo *getInfo, XrActionStatePose *data) { + OXR_TRACE_MARKER(); + struct oxr_session *sess = NULL; struct oxr_action *act = NULL; struct oxr_subaction_paths subaction_paths = {0}; @@ -523,6 +552,8 @@ oxr_xrEnumerateBoundSourcesForAction(XrSession session, uint32_t *sourceCountOutput, XrPath *sources) { + OXR_TRACE_MARKER(); + struct oxr_session *sess = NULL; struct oxr_action *act = NULL; struct oxr_logger log; @@ -552,6 +583,8 @@ oxr_xrApplyHapticFeedback(XrSession session, const XrHapticActionInfo *hapticActionInfo, const XrHapticBaseHeader *hapticEvent) { + OXR_TRACE_MARKER(); + struct oxr_session *sess = NULL; struct oxr_action *act = NULL; struct oxr_subaction_paths subaction_paths = {0}; @@ -578,6 +611,8 @@ oxr_xrApplyHapticFeedback(XrSession session, XrResult oxr_xrStopHapticFeedback(XrSession session, const XrHapticActionInfo *hapticActionInfo) { + OXR_TRACE_MARKER(); + struct oxr_session *sess = NULL; struct oxr_action *act = NULL; struct oxr_subaction_paths subaction_paths = {0}; diff --git a/src/xrt/state_trackers/oxr/oxr_api_debug.c b/src/xrt/state_trackers/oxr/oxr_api_debug.c index 738533983..9af50a0e1 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_debug.c +++ b/src/xrt/state_trackers/oxr/oxr_api_debug.c @@ -7,6 +7,8 @@ * @ingroup oxr_api */ +#include "util/u_trace_marker.h" + #include "oxr_objects.h" #include "oxr_logger.h" @@ -18,6 +20,8 @@ XrResult oxr_xrSetDebugUtilsObjectNameEXT(XrInstance instance, const XrDebugUtilsObjectNameInfoEXT *nameInfo) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrSetDebugUtilsObjectNameEXT"); @@ -30,6 +34,8 @@ oxr_xrCreateDebugUtilsMessengerEXT(XrInstance instance, const XrDebugUtilsMessengerCreateInfoEXT *createInfo, XrDebugUtilsMessengerEXT *messenger) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_debug_messenger *mssngr; struct oxr_logger log; @@ -52,6 +58,8 @@ oxr_xrCreateDebugUtilsMessengerEXT(XrInstance instance, XrResult oxr_xrDestroyDebugUtilsMessengerEXT(XrDebugUtilsMessengerEXT messenger) { + OXR_TRACE_MARKER(); + struct oxr_debug_messenger *mssngr; struct oxr_logger log; OXR_VERIFY_MESSENGER_AND_INIT_LOG(&log, messenger, mssngr, "xrDestroyDebugUtilsMessengerEXT"); @@ -66,6 +74,8 @@ oxr_xrSubmitDebugUtilsMessageEXT(XrInstance instance, XrDebugUtilsMessageTypeFlagsEXT messageTypes, const XrDebugUtilsMessengerCallbackDataEXT *callbackData) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrSubmitDebugUtilsMessageEXT"); @@ -78,6 +88,8 @@ oxr_xrSubmitDebugUtilsMessageEXT(XrInstance instance, XrResult oxr_xrSessionBeginDebugUtilsLabelRegionEXT(XrSession session, const XrDebugUtilsLabelEXT *labelInfo) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrSessionBeginDebugUtilsLabelRegionEXT"); @@ -89,6 +101,8 @@ oxr_xrSessionBeginDebugUtilsLabelRegionEXT(XrSession session, const XrDebugUtils XrResult oxr_xrSessionEndDebugUtilsLabelRegionEXT(XrSession session) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrSessionEndDebugUtilsLabelRegionEXT"); @@ -100,6 +114,8 @@ oxr_xrSessionEndDebugUtilsLabelRegionEXT(XrSession session) XrResult oxr_xrSessionInsertDebugUtilsLabelEXT(XrSession session, const XrDebugUtilsLabelEXT *labelInfo) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrSessionInsertDebugUtilsLabelEXT"); diff --git a/src/xrt/state_trackers/oxr/oxr_api_funcs.h b/src/xrt/state_trackers/oxr/oxr_api_funcs.h index e91d08161..59dc680c1 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_funcs.h +++ b/src/xrt/state_trackers/oxr/oxr_api_funcs.h @@ -50,6 +50,12 @@ oxr_xrEnumerateApiLayerProperties(uint32_t propertyCapacityInput, * */ +#ifdef OXR_HAVE_KHR_loader_init +//! OpenXR API function @ep{xrInitializeLoaderKHR} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrInitializeLoaderKHR(const XrLoaderInitInfoBaseHeaderKHR *loaderInitInfo); +#endif // OXR_HAVE_KHR_loader_init + //! OpenXR API function @ep{xrEnumerateInstanceExtensionProperties} XRAPI_ATTR XrResult XRAPI_CALL oxr_xrEnumerateInstanceExtensionProperties(const char *layerName, diff --git a/src/xrt/state_trackers/oxr/oxr_api_instance.c b/src/xrt/state_trackers/oxr/oxr_api_instance.c index 2f0e0b137..54d3b1ff8 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_instance.c +++ b/src/xrt/state_trackers/oxr/oxr_api_instance.c @@ -10,6 +10,7 @@ #include "xrt/xrt_compiler.h" #include "util/u_debug.h" +#include "util/u_trace_marker.h" #include "oxr_objects.h" #include "oxr_logger.h" @@ -20,6 +21,11 @@ #include "oxr_api_funcs.h" #include "oxr_api_verify.h" + +#ifdef XRT_OS_ANDROID +#include "android/android_globals.h" +#endif + #include "openxr/openxr.h" #include "openxr/openxr_reflection.h" @@ -39,6 +45,8 @@ oxr_xrEnumerateInstanceExtensionProperties(const char *layerName, uint32_t *propertyCountOutput, XrExtensionProperties *properties) { + OXR_TRACE_MARKER(); + struct oxr_logger log; oxr_log_init(&log, "xrEnumerateInstanceExtensionProperties"); @@ -46,6 +54,42 @@ oxr_xrEnumerateInstanceExtensionProperties(const char *layerName, ARRAY_SIZE(extension_properties), extension_properties, XR_SUCCESS); } +#ifdef OXR_HAVE_KHR_loader_init +XrResult +oxr_xrInitializeLoaderKHR(const XrLoaderInitInfoBaseHeaderKHR *loaderInitInfo) +{ + struct oxr_logger log; + oxr_log_init(&log, "oxr_xrInitializeLoaderKHR"); + + + oxr_log(&log, "Loader forwarded call to xrInitializeLoaderKHR."); +#ifdef XRT_OS_ANDROID + const XrLoaderInitInfoAndroidKHR *initInfoAndroid = + OXR_GET_INPUT_FROM_CHAIN(loaderInitInfo, XR_TYPE_LOADER_INIT_INFO_ANDROID_KHR, XrLoaderInitInfoAndroidKHR); + if (initInfoAndroid == NULL) { + return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, + "(loaderInitInfo) " + "Did not find XrLoaderInitInfoAndroidKHR"); + } + if (initInfoAndroid->applicationVM == NULL) { + return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, + "(initInfoAndroid->applicationVM) " + "applicationVM must be populated"); + } + if (initInfoAndroid->applicationContext == NULL) { + return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, + "(initInfoAndroid->applicationContext) " + "applicationContext must be populated"); + } + //! @todo check that applicationContext is in fact an Activity. + android_globals_store_vm_and_context(initInfoAndroid->applicationVM, initInfoAndroid->applicationContext); + +#endif // XRT_OS_ANDROID + return XR_SUCCESS; +} +#endif // OXR_HAVE_KHR_loader_init + + #ifdef XRT_OS_ANDROID static XrResult oxr_check_android_extensions(struct oxr_logger *log, const XrInstanceCreateInfo *createInfo) @@ -61,9 +105,8 @@ oxr_check_android_extensions(struct oxr_logger *log, const XrInstanceCreateInfo if (!foundAndroidExtension) { return oxr_error(log, XR_ERROR_INITIALIZATION_FAILED, "(createInfo->enabledExtensionNames) " - "Mandatory platform-specific " - "extension" - " " XR_KHR_ANDROID_CREATE_INSTANCE_EXTENSION_NAME " not specified"); + "Mandatory platform-specific extension " XR_KHR_ANDROID_CREATE_INSTANCE_EXTENSION_NAME + " not specified"); } { @@ -94,6 +137,8 @@ oxr_check_android_extensions(struct oxr_logger *log, const XrInstanceCreateInfo XrResult oxr_xrCreateInstance(const XrInstanceCreateInfo *createInfo, XrInstance *out_instance) { + OXR_TRACE_MARKER(); + XrResult ret; struct oxr_logger log; oxr_log_init(&log, "xrCreateInstance"); @@ -160,6 +205,8 @@ oxr_xrCreateInstance(const XrInstanceCreateInfo *createInfo, XrInstance *out_ins XrResult oxr_xrDestroyInstance(XrInstance instance) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrDestroyInstance"); @@ -170,6 +217,8 @@ oxr_xrDestroyInstance(XrInstance instance) XrResult oxr_xrGetInstanceProperties(XrInstance instance, XrInstanceProperties *instanceProperties) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetInstanceProperties"); @@ -180,6 +229,8 @@ oxr_xrGetInstanceProperties(XrInstance instance, XrInstanceProperties *instanceP XrResult oxr_xrPollEvent(XrInstance instance, XrEventDataBuffer *eventData) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrPollEvent"); @@ -191,6 +242,8 @@ oxr_xrPollEvent(XrInstance instance, XrEventDataBuffer *eventData) XrResult oxr_xrResultToString(XrInstance instance, XrResult value, char buffer[XR_MAX_RESULT_STRING_SIZE]) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrResultToString"); @@ -211,6 +264,8 @@ oxr_xrResultToString(XrInstance instance, XrResult value, char buffer[XR_MAX_RES XrResult oxr_xrStructureTypeToString(XrInstance instance, XrStructureType value, char buffer[XR_MAX_STRUCTURE_NAME_SIZE]) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrStructureTypeToString"); @@ -229,6 +284,8 @@ oxr_xrStructureTypeToString(XrInstance instance, XrStructureType value, char buf XrResult oxr_xrStringToPath(XrInstance instance, const char *pathString, XrPath *out_path) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; XrResult ret; @@ -254,6 +311,8 @@ XrResult oxr_xrPathToString( XrInstance instance, XrPath path, uint32_t bufferCapacityInput, uint32_t *bufferCountOutput, char *buffer) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; const char *str; @@ -283,6 +342,8 @@ oxr_xrPathToString( XrResult oxr_xrConvertTimespecTimeToTimeKHR(XrInstance instance, const struct timespec *timespecTime, XrTime *time) { + OXR_TRACE_MARKER(); + //! @todo do we need to check and see if this extension was //! enabled first? struct oxr_instance *inst; @@ -297,6 +358,8 @@ oxr_xrConvertTimespecTimeToTimeKHR(XrInstance instance, const struct timespec *t XrResult oxr_xrConvertTimeToTimespecTimeKHR(XrInstance instance, XrTime time, struct timespec *timespecTime) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrConvertTimeToTimespecTimeKHR"); diff --git a/src/xrt/state_trackers/oxr/oxr_api_negotiate.c b/src/xrt/state_trackers/oxr/oxr_api_negotiate.c index 91b27ab73..a97d643c7 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_negotiate.c +++ b/src/xrt/state_trackers/oxr/oxr_api_negotiate.c @@ -220,9 +220,11 @@ handle_non_null(struct oxr_instance *inst, struct oxr_logger *log, const char *n ENTRY_IF_EXT(xrThermalGetTemperatureTrendEXT, EXT_thermal_query); #endif // OXR_HAVE_EXT_thermal_query +#ifdef OXR_HAVE_EXT_hand_tracking ENTRY_IF_EXT(xrCreateHandTrackerEXT, EXT_hand_tracking); ENTRY_IF_EXT(xrDestroyHandTrackerEXT, EXT_hand_tracking); ENTRY_IF_EXT(xrLocateHandJointsEXT, EXT_hand_tracking); +#endif #if 0 #ifdef OXR_HAVE_EXT_debug_utils @@ -276,6 +278,10 @@ handle_null(struct oxr_logger *log, const char *name, PFN_xrVoidFunction *out_fu ENTRY(xrEnumerateInstanceExtensionProperties); ENTRY(xrEnumerateApiLayerProperties); +#ifdef OXR_HAVE_KHR_loader_init + ENTRY(xrInitializeLoaderKHR); +#endif // OXR_HAVE_KHR_loader_init + /* * This is fine to log, since there should not be other * null-instance calls. diff --git a/src/xrt/state_trackers/oxr/oxr_api_session.c b/src/xrt/state_trackers/oxr/oxr_api_session.c index d405d7f9d..970c72416 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_session.c +++ b/src/xrt/state_trackers/oxr/oxr_api_session.c @@ -15,6 +15,7 @@ #include "xrt/xrt_compiler.h" #include "util/u_debug.h" +#include "util/u_trace_marker.h" #include "oxr_objects.h" #include "oxr_logger.h" @@ -28,6 +29,8 @@ XrResult oxr_xrCreateSession(XrInstance instance, const XrSessionCreateInfo *createInfo, XrSession *out_session) { + OXR_TRACE_MARKER(); + XrResult ret; struct oxr_instance *inst; struct oxr_session *sess, **link; @@ -59,6 +62,8 @@ oxr_xrCreateSession(XrInstance instance, const XrSessionCreateInfo *createInfo, XrResult oxr_xrDestroySession(XrSession session) { + OXR_TRACE_MARKER(); + struct oxr_session *sess, **link; struct oxr_instance *inst; struct oxr_logger log; @@ -78,6 +83,8 @@ oxr_xrDestroySession(XrSession session) XrResult oxr_xrBeginSession(XrSession session, const XrSessionBeginInfo *beginInfo) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrBeginSession"); @@ -90,6 +97,8 @@ oxr_xrBeginSession(XrSession session, const XrSessionBeginInfo *beginInfo) XrResult oxr_xrEndSession(XrSession session) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrEndSession"); @@ -100,6 +109,8 @@ oxr_xrEndSession(XrSession session) XrResult oxr_xrWaitFrame(XrSession session, const XrFrameWaitInfo *frameWaitInfo, XrFrameState *frameState) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrWaitFrame"); @@ -113,6 +124,8 @@ oxr_xrWaitFrame(XrSession session, const XrFrameWaitInfo *frameWaitInfo, XrFrame XrResult oxr_xrBeginFrame(XrSession session, const XrFrameBeginInfo *frameBeginInfo) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrBeginFrame"); @@ -125,6 +138,8 @@ oxr_xrBeginFrame(XrSession session, const XrFrameBeginInfo *frameBeginInfo) XrResult oxr_xrEndFrame(XrSession session, const XrFrameEndInfo *frameEndInfo) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrEndFrame"); @@ -136,6 +151,8 @@ oxr_xrEndFrame(XrSession session, const XrFrameEndInfo *frameEndInfo) XrResult oxr_xrRequestExitSession(XrSession session) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrRequestExitSession"); @@ -151,6 +168,8 @@ oxr_xrLocateViews(XrSession session, uint32_t *viewCountOutput, XrView *views) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_space *spc; struct oxr_logger log; @@ -165,7 +184,14 @@ oxr_xrLocateViews(XrSession session, OXR_VERIFY_ARG_NOT_NULL(&log, views); } - return oxr_session_views(&log, sess, viewLocateInfo, viewState, viewCapacityInput, viewCountOutput, views); + return oxr_session_locate_views( // + &log, // + sess, // + viewLocateInfo, // + viewState, // + viewCapacityInput, // + viewCountOutput, // + views); // } @@ -184,6 +210,8 @@ oxr_xrGetVisibilityMaskKHR(XrSession session, XrVisibilityMaskTypeKHR visibilityMaskType, XrVisibilityMaskKHR *visibilityMask) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrGetVisibilityMaskKHR"); @@ -207,6 +235,8 @@ oxr_xrPerfSettingsSetPerformanceLevelEXT(XrSession session, XrPerfSettingsDomainEXT domain, XrPerfSettingsLevelEXT level) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrPerfSettingsSetPerformanceLevelEXT"); @@ -232,6 +262,8 @@ oxr_xrThermalGetTemperatureTrendEXT(XrSession session, float *tempHeadroom, float *tempSlope) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrThermalGetTemperatureTrendEXT"); @@ -239,12 +271,16 @@ oxr_xrThermalGetTemperatureTrendEXT(XrSession session, return oxr_error(&log, XR_ERROR_HANDLE_INVALID, "Not implemented"); } +#endif + /* * * XR_EXT_hand_tracking * */ +#ifdef XR_EXT_hand_tracking + static XrResult oxr_hand_tracker_destroy_cb(struct oxr_logger *log, struct oxr_handle_base *hb) { @@ -273,11 +309,12 @@ oxr_hand_tracker_create(struct oxr_logger *log, hand_tracker->hand = createInfo->hand; hand_tracker->hand_joint_set = createInfo->handJointSet; - //! @todo: Implement choice when more than one device has hand tracking + //! @todo: move hand tracking device selection to oxr_system. + // if no xdev with hand tracking is found, create hand tracker without xdev. for (uint32_t i = 0; i < sess->sys->num_xdevs; i++) { struct xrt_device *xdev = sess->sys->xdevs[i]; - if (!xdev->hand_tracking_supported) { + if (!xdev || !xdev->hand_tracking_supported) { continue; } @@ -309,6 +346,8 @@ oxr_xrCreateHandTrackerEXT(XrSession session, const XrHandTrackerCreateInfoEXT *createInfo, XrHandTrackerEXT *handTracker) { + OXR_TRACE_MARKER(); + struct oxr_hand_tracker *hand_tracker = NULL; struct oxr_session *sess = NULL; struct oxr_logger log; @@ -341,6 +380,8 @@ oxr_xrCreateHandTrackerEXT(XrSession session, XrResult oxr_xrDestroyHandTrackerEXT(XrHandTrackerEXT handTracker) { + OXR_TRACE_MARKER(); + struct oxr_hand_tracker *hand_tracker; struct oxr_logger log; OXR_VERIFY_HAND_TRACKER_AND_INIT_LOG(&log, handTracker, hand_tracker, "xrDestroyHandTrackerEXT"); @@ -353,6 +394,8 @@ oxr_xrLocateHandJointsEXT(XrHandTrackerEXT handTracker, const XrHandJointsLocateInfoEXT *locateInfo, XrHandJointLocationsEXT *locations) { + OXR_TRACE_MARKER(); + struct oxr_hand_tracker *hand_tracker; struct oxr_space *spc; struct oxr_logger log; diff --git a/src/xrt/state_trackers/oxr/oxr_api_space.c b/src/xrt/state_trackers/oxr/oxr_api_space.c index abd65c4ca..9d31e70ba 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_space.c +++ b/src/xrt/state_trackers/oxr/oxr_api_space.c @@ -9,8 +9,10 @@ #include "xrt/xrt_compiler.h" -#include "math/m_api.h" #include "util/u_debug.h" +#include "util/u_trace_marker.h" + +#include "math/m_api.h" #include "oxr_objects.h" #include "oxr_logger.h" @@ -28,6 +30,8 @@ XrResult oxr_xrCreateActionSpace(XrSession session, const XrActionSpaceCreateInfo *createInfo, XrSpace *space) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_action *act; struct oxr_logger log; @@ -59,6 +63,8 @@ oxr_xrEnumerateReferenceSpaces(XrSession session, uint32_t *spaceCountOutput, XrReferenceSpaceType *spaces) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrEnumerateReferenceSpaces"); @@ -70,6 +76,8 @@ oxr_xrEnumerateReferenceSpaces(XrSession session, XrResult oxr_xrGetReferenceSpaceBoundsRect(XrSession session, XrReferenceSpaceType referenceSpaceType, XrExtent2Df *bounds) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrGetReferenceSpaceBoundsRect"); @@ -97,6 +105,8 @@ oxr_xrGetReferenceSpaceBoundsRect(XrSession session, XrReferenceSpaceType refere XrResult oxr_xrCreateReferenceSpace(XrSession session, const XrReferenceSpaceCreateInfo *createInfo, XrSpace *out_space) { + OXR_TRACE_MARKER(); + XrResult ret; struct oxr_session *sess; struct oxr_space *spc = NULL; @@ -118,6 +128,8 @@ oxr_xrCreateReferenceSpace(XrSession session, const XrReferenceSpaceCreateInfo * XrResult oxr_xrLocateSpace(XrSpace space, XrSpace baseSpace, XrTime time, XrSpaceLocation *location) { + OXR_TRACE_MARKER(); + struct oxr_space *spc; struct oxr_space *baseSpc; struct oxr_logger log; @@ -137,6 +149,8 @@ oxr_xrLocateSpace(XrSpace space, XrSpace baseSpace, XrTime time, XrSpaceLocation XrResult oxr_xrDestroySpace(XrSpace space) { + OXR_TRACE_MARKER(); + struct oxr_space *spc; struct oxr_logger log; OXR_VERIFY_SPACE_AND_INIT_LOG(&log, space, spc, "xrDestroySpace"); diff --git a/src/xrt/state_trackers/oxr/oxr_api_swapchain.c b/src/xrt/state_trackers/oxr/oxr_api_swapchain.c index 2b97b96d8..9677b4fd7 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_swapchain.c +++ b/src/xrt/state_trackers/oxr/oxr_api_swapchain.c @@ -10,6 +10,7 @@ #include "xrt/xrt_compiler.h" #include "util/u_debug.h" +#include "util/u_trace_marker.h" #include "oxr_objects.h" #include "oxr_logger.h" @@ -30,6 +31,8 @@ oxr_xrEnumerateSwapchainFormats(XrSession session, uint32_t *formatCountOutput, int64_t *formats) { + OXR_TRACE_MARKER(); + struct oxr_session *sess; struct oxr_logger log; OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrEnumerateSwapchainFormats"); @@ -40,6 +43,8 @@ oxr_xrEnumerateSwapchainFormats(XrSession session, XrResult oxr_xrCreateSwapchain(XrSession session, const XrSwapchainCreateInfo *createInfo, XrSwapchain *out_swapchain) { + OXR_TRACE_MARKER(); + XrResult ret; struct oxr_session *sess; struct oxr_swapchain *sc; @@ -67,13 +72,15 @@ oxr_xrCreateSwapchain(XrSession session, const XrSwapchainCreateInfo *createInfo flags |= XR_SWAPCHAIN_USAGE_TRANSFER_DST_BIT; flags |= XR_SWAPCHAIN_USAGE_SAMPLED_BIT; flags |= XR_SWAPCHAIN_USAGE_MUTABLE_FORMAT_BIT; - if (inst->extensions.MND_swapchain_usage_input_attachment_bit) { - flags |= XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_MND; + if (inst->extensions.MND_swapchain_usage_input_attachment_bit || + inst->extensions.KHR_swapchain_usage_input_attachment_bit) { + // aliased to XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_MND + flags |= XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_KHR; } if ((createInfo->usageFlags & ~flags) != 0) { return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, - "(createInfo->usageFlags == 0x08%" PRIx64 ") contains invalid flags", + "(createInfo->usageFlags == 0x%04" PRIx64 ") contains invalid flags", createInfo->usageFlags); } bool format_supported = false; @@ -87,7 +94,7 @@ oxr_xrCreateSwapchain(XrSession session, const XrSwapchainCreateInfo *createInfo if (!format_supported) { return oxr_error(&log, XR_ERROR_SWAPCHAIN_FORMAT_UNSUPPORTED, - "(createInfo->format == 0x08%" PRIx64 ") is not supported", createInfo->format); + "(createInfo->format == 0x%04" PRIx64 ") is not supported", createInfo->format); } ret = sess->create_swapchain(&log, sess, createInfo, &sc); @@ -103,6 +110,8 @@ oxr_xrCreateSwapchain(XrSession session, const XrSwapchainCreateInfo *createInfo XrResult oxr_xrDestroySwapchain(XrSwapchain swapchain) { + OXR_TRACE_MARKER(); + struct oxr_swapchain *sc; struct oxr_logger log; OXR_VERIFY_SWAPCHAIN_AND_INIT_LOG(&log, swapchain, sc, "xrDestroySwapchain"); @@ -116,6 +125,8 @@ oxr_xrEnumerateSwapchainImages(XrSwapchain swapchain, uint32_t *imageCountOutput, XrSwapchainImageBaseHeader *images) { + OXR_TRACE_MARKER(); + struct oxr_swapchain *sc; struct oxr_logger log; OXR_VERIFY_SWAPCHAIN_AND_INIT_LOG(&log, swapchain, sc, "xrEnumerateSwapchainImages"); @@ -137,6 +148,8 @@ oxr_xrEnumerateSwapchainImages(XrSwapchain swapchain, XrResult oxr_xrAcquireSwapchainImage(XrSwapchain swapchain, const XrSwapchainImageAcquireInfo *acquireInfo, uint32_t *index) { + OXR_TRACE_MARKER(); + struct oxr_swapchain *sc; struct oxr_logger log; OXR_VERIFY_SWAPCHAIN_AND_INIT_LOG(&log, swapchain, sc, "xrAcquireSwapchainImage"); @@ -149,6 +162,8 @@ oxr_xrAcquireSwapchainImage(XrSwapchain swapchain, const XrSwapchainImageAcquire XrResult oxr_xrWaitSwapchainImage(XrSwapchain swapchain, const XrSwapchainImageWaitInfo *waitInfo) { + OXR_TRACE_MARKER(); + struct oxr_swapchain *sc; struct oxr_logger log; OXR_VERIFY_SWAPCHAIN_AND_INIT_LOG(&log, swapchain, sc, "xrWaitSwapchainImage"); @@ -160,6 +175,8 @@ oxr_xrWaitSwapchainImage(XrSwapchain swapchain, const XrSwapchainImageWaitInfo * XrResult oxr_xrReleaseSwapchainImage(XrSwapchain swapchain, const XrSwapchainImageReleaseInfo *releaseInfo) { + OXR_TRACE_MARKER(); + struct oxr_swapchain *sc; struct oxr_logger log; OXR_VERIFY_SWAPCHAIN_AND_INIT_LOG(&log, swapchain, sc, "xrReleaseSwapchainImage"); diff --git a/src/xrt/state_trackers/oxr/oxr_api_system.c b/src/xrt/state_trackers/oxr/oxr_api_system.c index 67388c714..ef3f24e5f 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_system.c +++ b/src/xrt/state_trackers/oxr/oxr_api_system.c @@ -15,7 +15,9 @@ #include "xrt/xrt_compiler.h" #include "xrt/xrt_gfx_gl.h" #include "xrt/xrt_gfx_gles.h" + #include "util/u_debug.h" +#include "util/u_trace_marker.h" #include "oxr_objects.h" #include "oxr_logger.h" @@ -41,6 +43,8 @@ XrResult oxr_xrGetSystem(XrInstance instance, const XrSystemGetInfo *getInfo, XrSystemId *systemId) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetSystem"); @@ -64,6 +68,8 @@ oxr_xrGetSystem(XrInstance instance, const XrSystemGetInfo *getInfo, XrSystemId XrResult oxr_xrGetSystemProperties(XrInstance instance, XrSystemId systemId, XrSystemProperties *properties) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetSystemProperties"); @@ -80,6 +86,8 @@ oxr_xrEnumerateViewConfigurations(XrInstance instance, uint32_t *viewConfigurationTypeCountOutput, XrViewConfigurationType *viewConfigurationTypes) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrEnumerateViewConfigurations"); @@ -97,6 +105,8 @@ oxr_xrEnumerateEnvironmentBlendModes(XrInstance instance, uint32_t *environmentBlendModeCountOutput, XrEnvironmentBlendMode *environmentBlendModes) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrEnumerateEnvironmentBlendModes"); @@ -120,6 +130,8 @@ oxr_xrGetViewConfigurationProperties(XrInstance instance, XrViewConfigurationType viewConfigurationType, XrViewConfigurationProperties *configurationProperties) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetViewConfigurationProperties"); @@ -137,6 +149,8 @@ oxr_xrEnumerateViewConfigurationViews(XrInstance instance, uint32_t *viewCountOutput, XrViewConfigurationView *views) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrEnumerateViewConfigurationViews"); @@ -160,6 +174,8 @@ oxr_xrGetOpenGLESGraphicsRequirementsKHR(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsOpenGLESKHR *graphicsRequirements) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetOpenGLESGraphicsRequirementsKHR"); @@ -194,6 +210,8 @@ oxr_xrGetOpenGLGraphicsRequirementsKHR(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsOpenGLKHR *graphicsRequirements) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetOpenGLGraphicsRequirementsKHR"); @@ -230,6 +248,8 @@ oxr_xrGetVulkanInstanceExtensionsKHR(XrInstance instance, uint32_t *namesCountOutput, char *namesString) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetVulkanInstanceExtensionsKHR"); @@ -245,6 +265,8 @@ oxr_xrGetVulkanDeviceExtensionsKHR(XrInstance instance, uint32_t *namesCountOutput, char *namesString) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetVulkanDeviceExtensionsKHR"); @@ -263,6 +285,8 @@ oxr_xrGetVulkanGraphicsDeviceKHR(XrInstance instance, VkInstance vkInstance, VkPhysicalDevice *vkPhysicalDevice) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetVulkanGraphicsDeviceKHR"); @@ -277,6 +301,8 @@ oxr_xrGetVulkanGraphicsDevice2KHR(XrInstance instance, const XrVulkanGraphicsDeviceGetInfoKHR *getInfo, VkPhysicalDevice *vkPhysicalDevice) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetVulkanGraphicsDeviceKHR"); @@ -294,6 +320,8 @@ oxr_xrGetVulkanGraphicsRequirementsKHR(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsVulkanKHR *graphicsRequirements) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetVulkanGraphicsRequirementsKHR"); @@ -308,6 +336,8 @@ oxr_xrGetVulkanGraphicsRequirements2KHR(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsVulkan2KHR *graphicsRequirements) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetVulkanGraphicsRequirementsKHR"); @@ -325,6 +355,8 @@ oxr_xrCreateVulkanInstanceKHR(XrInstance instance, VkInstance *vulkanInstance, VkResult *vulkanResult) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrCreateVulkanInstanceKHR"); @@ -354,6 +386,8 @@ oxr_xrCreateVulkanDeviceKHR(XrInstance instance, VkDevice *vulkanDevice, VkResult *vulkanResult) { + OXR_TRACE_MARKER(); + struct oxr_instance *inst; struct oxr_logger log; @@ -369,10 +403,10 @@ oxr_xrCreateVulkanDeviceKHR(XrInstance instance, // VK_NULL_HANDLE is 0 OXR_VERIFY_ARG_NOT_NULL(&log, createInfo->vulkanPhysicalDevice); - OXR_VERIFY_ARG_NOT_NULL(&log, sys->vulkan_enable2_physical_device); + OXR_VERIFY_ARG_NOT_NULL(&log, sys->suggested_vulkan_physical_device); OXR_VERIFY_ARG_NOT_NULL(&log, sys->vulkan_enable2_instance); - if (sys->vulkan_enable2_physical_device != createInfo->vulkanPhysicalDevice) { + if (sys->suggested_vulkan_physical_device != createInfo->vulkanPhysicalDevice) { return oxr_error(&log, XR_ERROR_HANDLE_INVALID, "createInfo->vulkanPhysicalDevice must be the device " "returned by xrGetVulkanGraphicsDeviceKHR"); diff --git a/src/xrt/state_trackers/oxr/oxr_api_verify.h b/src/xrt/state_trackers/oxr/oxr_api_verify.h index bd6f63fb1..2c05af55a 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_verify.h +++ b/src/xrt/state_trackers/oxr/oxr_api_verify.h @@ -44,7 +44,7 @@ extern "C" { /*! - * @ingroup oxr_api + * @addtogroup oxr_api * @{ */ diff --git a/src/xrt/state_trackers/oxr/oxr_binding.c b/src/xrt/state_trackers/oxr/oxr_binding.c index 168c8b1fc..c019f6e28 100644 --- a/src/xrt/state_trackers/oxr/oxr_binding.c +++ b/src/xrt/state_trackers/oxr/oxr_binding.c @@ -323,7 +323,37 @@ oxr_find_profile_for_device(struct oxr_logger *log, interaction_profile_find(log, inst, inst->path_cache.khr_simple_controller, out_p); interaction_profile_find(log, inst, inst->path_cache.htc_vive_controller, out_p); return; - default: return; + case XRT_DEVICE_TOUCH_CONTROLLER: + interaction_profile_find(log, inst, inst->path_cache.khr_simple_controller, out_p); + interaction_profile_find(log, inst, inst->path_cache.oculus_touch_controller, out_p); + return; + case XRT_DEVICE_WMR_CONTROLLER: + interaction_profile_find(log, inst, inst->path_cache.khr_simple_controller, out_p); + interaction_profile_find(log, inst, inst->path_cache.microsoft_motion_controller, out_p); + return; + case XRT_DEVICE_GO_CONTROLLER: + interaction_profile_find(log, inst, inst->path_cache.khr_simple_controller, out_p); + interaction_profile_find(log, inst, inst->path_cache.oculus_go_controller, out_p); + return; + case XRT_DEVICE_VIVE_PRO: + interaction_profile_find(log, inst, inst->path_cache.khr_simple_controller, out_p); + interaction_profile_find(log, inst, inst->path_cache.htc_vive_pro, out_p); + return; + case XRT_DEVICE_XBOX_CONTROLLER: + interaction_profile_find(log, inst, inst->path_cache.khr_simple_controller, out_p); + interaction_profile_find(log, inst, inst->path_cache.microsoft_xbox_controller, out_p); + return; + case XRT_DEVICE_HAND_INTERACTION: + interaction_profile_find(log, inst, inst->path_cache.khr_simple_controller, out_p); + interaction_profile_find(log, inst, inst->path_cache.msft_hand_interaction, out_p); + return; + + // no interaction + case XRT_DEVICE_GENERIC_HMD: + case XRT_DEVICE_REALSENSE: + case XRT_DEVICE_HAND_TRACKER: + case XRT_DEVICE_VIVE_TRACKER_GEN1: + case XRT_DEVICE_VIVE_TRACKER_GEN2: return; } } @@ -331,7 +361,7 @@ void oxr_binding_find_bindings_from_key(struct oxr_logger *log, struct oxr_interaction_profile *p, uint32_t key, - struct oxr_binding *bindings[32], + struct oxr_binding *bindings[OXR_MAX_BINDINGS_PER_ACTION], size_t *num_bindings) { if (p == NULL) { diff --git a/src/xrt/state_trackers/oxr/oxr_chain.h b/src/xrt/state_trackers/oxr/oxr_chain.h index 3f9e77032..33ceec053 100644 --- a/src/xrt/state_trackers/oxr/oxr_chain.h +++ b/src/xrt/state_trackers/oxr/oxr_chain.h @@ -15,7 +15,7 @@ extern "C" { #endif /*! - * @ingroup oxr_api + * @addtogroup oxr_api * @{ */ diff --git a/src/xrt/state_trackers/oxr/oxr_extension_support.h b/src/xrt/state_trackers/oxr/oxr_extension_support.h index f6f85d8c9..a34b14f8a 100644 --- a/src/xrt/state_trackers/oxr/oxr_extension_support.h +++ b/src/xrt/state_trackers/oxr/oxr_extension_support.h @@ -31,6 +31,28 @@ #endif +/* + * XR_KHR_loader_init + */ +#if defined(XR_KHR_loader_init) && defined(XR_USE_PLATFORM_ANDROID) +#define OXR_HAVE_KHR_loader_init +#define OXR_EXTENSION_SUPPORT_KHR_loader_init(_) _(KHR_loader_init, KHR_LOADER_INIT) +#else +#define OXR_EXTENSION_SUPPORT_KHR_loader_init(_) +#endif + + +/* + * XR_KHR_loader_init_android + */ +#if defined(XR_KHR_loader_init_android) && defined(OXR_HAVE_KHR_loader_init) && defined(XR_USE_PLATFORM_ANDROID) +#define OXR_HAVE_KHR_loader_init_android +#define OXR_EXTENSION_SUPPORT_KHR_loader_init_android(_) _(KHR_loader_init_android, KHR_LOADER_INIT_ANDROID) +#else +#define OXR_EXTENSION_SUPPORT_KHR_loader_init_android(_) +#endif + + /* * XR_KHR_convert_timespec_time */ @@ -178,6 +200,18 @@ #endif +/* + * XR_KHR_swapchain_usage_input_attachment_bit + */ +#if defined(XR_KHR_swapchain_usage_input_attachment_bit) +#define OXR_HAVE_KHR_swapchain_usage_input_attachment_bit +#define OXR_EXTENSION_SUPPORT_KHR_swapchain_usage_input_attachment_bit(_) \ + _(KHR_swapchain_usage_input_attachment_bit, KHR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT) +#else +#define OXR_EXTENSION_SUPPORT_KHR_swapchain_usage_input_attachment_bit(_) +#endif + + /* * XR_EXTX_overlay */ @@ -247,6 +281,8 @@ // clang-format off #define OXR_EXTENSION_SUPPORT_GENERATE(_) \ OXR_EXTENSION_SUPPORT_KHR_android_create_instance(_) \ + OXR_EXTENSION_SUPPORT_KHR_loader_init(_) \ + OXR_EXTENSION_SUPPORT_KHR_loader_init_android(_) \ OXR_EXTENSION_SUPPORT_KHR_convert_timespec_time(_) \ OXR_EXTENSION_SUPPORT_KHR_opengl_enable(_) \ OXR_EXTENSION_SUPPORT_KHR_opengl_es_enable(_) \ @@ -260,6 +296,7 @@ OXR_EXTENSION_SUPPORT_EXT_debug_utils(_) \ OXR_EXTENSION_SUPPORT_MND_headless(_) \ OXR_EXTENSION_SUPPORT_MND_swapchain_usage_input_attachment_bit(_) \ + OXR_EXTENSION_SUPPORT_KHR_swapchain_usage_input_attachment_bit(_) \ OXR_EXTENSION_SUPPORT_EXTX_overlay(_) \ OXR_EXTENSION_SUPPORT_MNDX_egl_enable(_) \ OXR_EXTENSION_SUPPORT_MNDX_ball_on_a_stick_controller(_) \ diff --git a/src/xrt/state_trackers/oxr/oxr_input.c b/src/xrt/state_trackers/oxr/oxr_input.c index 9bf8c70ad..b75ff9d52 100644 --- a/src/xrt/state_trackers/oxr/oxr_input.c +++ b/src/xrt/state_trackers/oxr/oxr_input.c @@ -1563,6 +1563,11 @@ oxr_action_enumerate_bound_sources(struct oxr_logger *log, for (uint32_t i = 0; i < act_attached->X.num_inputs; i++) { \ add_path_to_set(temp, act_attached->X.inputs[i].bound_path, &num_paths); \ } \ + } \ + if (act_attached->X.num_outputs > 0) { \ + for (uint32_t i = 0; i < act_attached->X.num_outputs; i++) { \ + add_path_to_set(temp, act_attached->X.outputs[i].bound_path, &num_paths); \ + } \ } OXR_FOR_EACH_SUBACTION_PATH(ACCUMULATE_PATHS) diff --git a/src/xrt/state_trackers/oxr/oxr_instance.c b/src/xrt/state_trackers/oxr/oxr_instance.c index 1600e0571..ddbdc6315 100644 --- a/src/xrt/state_trackers/oxr/oxr_instance.c +++ b/src/xrt/state_trackers/oxr/oxr_instance.c @@ -57,7 +57,7 @@ extern int oxr_sdl2_hack_create(void **out_hack); extern void -oxr_sdl2_hack_start(void *hack, struct xrt_instance *xinst); +oxr_sdl2_hack_start(void *hack, struct xrt_instance *xinst, struct xrt_device **xdevs); extern void oxr_sdl2_hack_stop(void **hack_ptr); @@ -121,6 +121,56 @@ min_size_t(size_t a, size_t b) return a < b ? a : b; } +static bool +starts_with(const char *with, const char *string) +{ + assert(with != NULL); + + if (string == NULL) { + return false; + } + + for (uint32_t i = 0; with[i] != 0; i++) { + if (string[i] != with[i]) { + return false; + } + } + + return true; +} + +static void +detect_engine(struct oxr_logger *log, struct oxr_instance *inst, const XrInstanceCreateInfo *createInfo) +{ + if (starts_with("UnrealEngine4", createInfo->applicationInfo.engineName)) { + inst->appinfo.detected.engine.name = "UnrealEngine"; + inst->appinfo.detected.engine.major = 4; + inst->appinfo.detected.engine.minor = (createInfo->applicationInfo.engineVersion >> 16) & 0xffff; + inst->appinfo.detected.engine.patch = createInfo->applicationInfo.engineVersion & 0xffff; + } + + if (starts_with("UnrealEngine5", createInfo->applicationInfo.engineName)) { + inst->appinfo.detected.engine.name = "UnrealEngine"; + inst->appinfo.detected.engine.major = 5; + inst->appinfo.detected.engine.minor = (createInfo->applicationInfo.engineVersion >> 16) & 0xffff; + inst->appinfo.detected.engine.patch = createInfo->applicationInfo.engineVersion & 0xffff; + } +} + +static void +apply_quirks(struct oxr_logger *log, struct oxr_instance *inst) +{ +#if 0 + // This is no longer needed. + if (starts_with("UnrealEngine", inst->appinfo.detected.engine.name) && // + inst->appinfo.detected.engine.major == 4 && // + inst->appinfo.detected.engine.minor <= 27 && // + inst->appinfo.detected.engine.patch <= 0) { + inst->quirks.disable_vulkan_format_depth_stencil = true; + } +#endif +} + XrResult oxr_instance_create(struct oxr_logger *log, const XrInstanceCreateInfo *createInfo, struct oxr_instance **out_instance) { @@ -184,6 +234,8 @@ oxr_instance_create(struct oxr_logger *log, const XrInstanceCreateInfo *createIn cache_path(log, inst, "/interaction_profiles/oculus/touch_controller", &inst->path_cache.oculus_touch_controller); cache_path(log, inst, "/interaction_profiles/valve/index_controller", &inst->path_cache.valve_index_controller); cache_path(log, inst, "/interaction_profiles/mndx/ball_on_a_stick_controller", &inst->path_cache.mndx_ball_on_a_stick_controller); + cache_path(log, inst, "/interaction_profiles/microsoft/hand_interaction", &inst->path_cache.msft_hand_interaction); + // clang-format on // fill in our application info - @todo - replicate all createInfo @@ -321,15 +373,39 @@ oxr_instance_create(struct oxr_logger *log, const XrInstanceCreateInfo *createIn inst->timekeeping = time_state_create(); - //! @todo check if this (and other creates) failed? + // Detect game engine. + detect_engine(log, inst, createInfo); + + // Apply any quirks + apply_quirks(log, inst); + u_var_add_root((void *)inst, "XrInstance", true); /* ---- HACK ---- */ - oxr_sdl2_hack_start(inst->hack, inst->xinst); + oxr_sdl2_hack_start(inst->hack, inst->xinst, sys->xdevs); /* ---- HACK ---- */ + oxr_log(log, + "Instance created\n" + "\tcreateInfo->applicationInfo.applicationName: %s\n" + "\tcreateInfo->applicationInfo.applicationVersion: %i\n" + "\tcreateInfo->applicationInfo.engineName: %s\n" + "\tcreateInfo->applicationInfo.engineVersion: %i\n" + "\tappinfo.detected.engine.name: %s\n" + "\tappinfo.detected.engine.version: %i.%i.%i\n" + "\tquirks.disable_vulkan_format_depth_stencil: %s", + createInfo->applicationInfo.applicationName, // + createInfo->applicationInfo.applicationVersion, // + createInfo->applicationInfo.engineName, // + createInfo->applicationInfo.engineVersion, // + inst->appinfo.detected.engine.name, // + inst->appinfo.detected.engine.major, // + inst->appinfo.detected.engine.minor, // + inst->appinfo.detected.engine.patch, // + inst->quirks.disable_vulkan_format_depth_stencil ? "true" : "false"); // + *out_instance = inst; return XR_SUCCESS; diff --git a/src/xrt/state_trackers/oxr/oxr_logger.h b/src/xrt/state_trackers/oxr/oxr_logger.h index b749861e1..969b64733 100644 --- a/src/xrt/state_trackers/oxr/oxr_logger.h +++ b/src/xrt/state_trackers/oxr/oxr_logger.h @@ -41,7 +41,7 @@ struct oxr_logger /*! - * @ingroup oxr_main + * @addtogroup oxr_main * @{ */ diff --git a/src/xrt/state_trackers/oxr/oxr_objects.h b/src/xrt/state_trackers/oxr/oxr_objects.h index f3c92461e..59fe8d67b 100644 --- a/src/xrt/state_trackers/oxr/oxr_objects.h +++ b/src/xrt/state_trackers/oxr/oxr_objects.h @@ -118,6 +118,7 @@ struct oxr_hand_tracker; #define XRT_MAX_HANDLE_CHILDREN 256 #define OXR_MAX_SWAPCHAIN_IMAGES 8 +#define OXR_MAX_BINDINGS_PER_ACTION 16 struct time_state; @@ -369,7 +370,7 @@ oxr_action_to_openxr(struct oxr_action *act) * @return false if an invalid subaction path is provided. * * @public @memberof oxr_instance - * @relatesalso oxr_sub_paths + * @see oxr_sub_paths */ bool oxr_classify_sub_action_paths(struct oxr_logger *log, @@ -410,7 +411,7 @@ oxr_action_create(struct oxr_logger *log, /*! * @public @memberof oxr_session - * @relatesalso oxr_action_set + * @see oxr_action_set */ XrResult oxr_session_attach_action_sets(struct oxr_logger *log, @@ -545,7 +546,7 @@ void oxr_binding_find_bindings_from_key(struct oxr_logger *log, struct oxr_interaction_profile *profile, uint32_t key, - struct oxr_binding *bindings[32], + struct oxr_binding *bindings[OXR_MAX_BINDINGS_PER_ACTION], size_t *num_bindings); /*! @@ -611,6 +612,12 @@ oxr_session_enumerate_formats(struct oxr_logger *log, uint32_t *formatCountOutput, int64_t *formats); +/*! + * Change the state of the session, queues a event. + */ +void +oxr_session_change_state(struct oxr_logger *log, struct oxr_session *sess, XrSessionState state); + XrResult oxr_session_begin(struct oxr_logger *log, struct oxr_session *sess, const XrSessionBeginInfo *beginInfo); @@ -634,13 +641,13 @@ oxr_session_get_view_relation_at(struct oxr_logger *, struct xrt_space_relation *out_relation); XrResult -oxr_session_views(struct oxr_logger *log, - struct oxr_session *sess, - const XrViewLocateInfo *viewLocateInfo, - XrViewState *viewState, - uint32_t viewCapacityInput, - uint32_t *viewCountOutput, - XrView *views); +oxr_session_locate_views(struct oxr_logger *log, + struct oxr_session *sess, + const XrViewLocateInfo *viewLocateInfo, + XrViewState *viewState, + uint32_t viewCapacityInput, + uint32_t *viewCountOutput, + XrView *views); XrResult oxr_session_frame_wait(struct oxr_logger *log, struct oxr_session *sess, XrFrameState *frameState); @@ -1092,9 +1099,13 @@ struct oxr_system uint32_t num_blend_modes; XrEnvironmentBlendMode blend_modes[3]; +#ifdef XR_USE_GRAPHICS_API_VULKAN //! The instance/device we create when vulkan_enable2 is used VkInstance vulkan_enable2_instance; - VkPhysicalDevice vulkan_enable2_physical_device; + //! The device returned with the last xrGetVulkanGraphicsDeviceKHR or xrGetVulkanGraphicsDevice2KHR call. + //! XR_NULL_HANDLE if neither has been called. + VkPhysicalDevice suggested_vulkan_physical_device; +#endif }; #define GET_XDEV_BY_ROLE(SYS, ROLE) SYS->role.ROLE == XRT_DEVICE_ROLE_UNASSIGNED ? NULL : SYS->xdevs[SYS->role.ROLE] @@ -1184,8 +1195,29 @@ struct oxr_instance XrPath oculus_touch_controller; XrPath valve_index_controller; XrPath mndx_ball_on_a_stick_controller; + XrPath msft_hand_interaction; } path_cache; + struct + { + struct + { + struct + { + uint32_t major; + uint32_t minor; + uint32_t patch; + const char *name; //< Engine name, not freed. + } engine; + } detected; + } appinfo; + + struct + { + //! Unreal has a bug in the VulkanRHI backend. + bool disable_vulkan_format_depth_stencil; + } quirks; + //! Debug messengers struct oxr_debug_messenger *messengers[XRT_MAX_HANDLE_CHILDREN]; @@ -1301,6 +1333,9 @@ struct oxr_session */ bool frame_timing_spew; + //! Extra sleep in wait frame. + uint32_t frame_timing_wait_sleep_ms; + /*! * To pipe swapchain creation to right code. */ @@ -1532,8 +1567,6 @@ struct oxr_action_output }; -#define OXR_MAX_BINDINGS_PER_ACTION 16 - /*! * The set of inputs/outputs for a single sub-action path for an action. * diff --git a/src/xrt/state_trackers/oxr/oxr_session.c b/src/xrt/state_trackers/oxr/oxr_session.c index 216abd2e1..7495d1f2f 100644 --- a/src/xrt/state_trackers/oxr/oxr_session.c +++ b/src/xrt/state_trackers/oxr/oxr_session.c @@ -1,9 +1,10 @@ -// Copyright 2018-2020, Collabora, Ltd. +// Copyright 2018-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file * @brief Holds session related functions. * @author Jakob Bornecrantz + * @author Moses Turner * @ingroup oxr_main */ @@ -24,6 +25,7 @@ #include "util/u_debug.h" #include "util/u_misc.h" #include "util/u_time.h" +#include "util/u_verify.h" #include "math/m_api.h" #include "math/m_mathinclude.h" @@ -44,6 +46,7 @@ DEBUG_GET_ONCE_NUM_OPTION(ipd, "OXR_DEBUG_IPD_MM", 63) +DEBUG_GET_ONCE_NUM_OPTION(wait_frame_sleep, "OXR_DEBUG_WAIT_FRAME_EXTRA_SLEEP_MS", 0) DEBUG_GET_ONCE_BOOL_OPTION(frame_timing_spew, "OXR_FRAME_TIMING_SPEW", false) #define CALL_CHK(call) \ @@ -86,7 +89,7 @@ to_string(XrSessionState state) } } -static void +void oxr_session_change_state(struct oxr_logger *log, struct oxr_session *sess, XrSessionState state) { oxr_event_push_XrEventDataSessionStateChanged(log, sess, state, 0); @@ -100,6 +103,7 @@ oxr_session_enumerate_formats(struct oxr_logger *log, uint32_t *formatCountOutput, int64_t *formats) { + struct oxr_instance *inst = sess->sys->inst; struct xrt_compositor *xc = sess->compositor; if (formatCountOutput == NULL) { return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "(formatCountOutput == NULL) can not be null"); @@ -111,8 +115,21 @@ oxr_session_enumerate_formats(struct oxr_logger *log, return oxr_session_success_result(sess); } - OXR_TWO_CALL_HELPER(log, formatCapacityInput, formatCountOutput, formats, xc->info.num_formats, - xc->info.formats, oxr_session_success_result(sess)); + uint32_t filtered_count = 0; + int64_t filtered_formats[XRT_MAX_SWAPCHAIN_FORMATS]; + for (uint32_t i = 0; i < xc->info.num_formats; i++) { + int64_t format = xc->info.formats[i]; + + if (inst->quirks.disable_vulkan_format_depth_stencil && + format == 130 /* VK_FORMAT_D32_SFLOAT_S8_UINT */) { + continue; + } + + filtered_formats[filtered_count++] = format; + } + + OXR_TWO_CALL_HELPER(log, formatCapacityInput, formatCountOutput, formats, filtered_count, filtered_formats, + oxr_session_success_result(sess)); } XrResult @@ -172,10 +189,10 @@ oxr_session_end(struct oxr_logger *log, struct oxr_session *sess) oxr_session_change_state(log, sess, XR_SESSION_STATE_IDLE); if (sess->exiting) { oxr_session_change_state(log, sess, XR_SESSION_STATE_EXITING); + } else { + oxr_session_change_state(log, sess, XR_SESSION_STATE_READY); } - oxr_session_change_state(log, sess, XR_SESSION_STATE_READY); - sess->has_begun = false; return oxr_session_success_result(sess); @@ -313,13 +330,13 @@ xrt_to_view_state_flags(enum xrt_space_relation_flags flags) } XrResult -oxr_session_views(struct oxr_logger *log, - struct oxr_session *sess, - const XrViewLocateInfo *viewLocateInfo, - XrViewState *viewState, - uint32_t viewCapacityInput, - uint32_t *viewCountOutput, - XrView *views) +oxr_session_locate_views(struct oxr_logger *log, + struct oxr_session *sess, + const XrViewLocateInfo *viewLocateInfo, + XrViewState *viewState, + uint32_t viewCapacityInput, + uint32_t *viewCountOutput, + XrView *views) { struct xrt_device *xdev = GET_XDEV_BY_ROLE(sess->sys, head); struct oxr_space *baseSpc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, viewLocateInfo->space); @@ -350,8 +367,13 @@ oxr_session_views(struct oxr_logger *log, // Get the viewLocateInfo->space to view space relation. struct xrt_space_relation pure_relation; - oxr_space_ref_relation(log, sess, XR_REFERENCE_SPACE_TYPE_VIEW, baseSpc->type, viewLocateInfo->displayTime, - &pure_relation); + oxr_space_ref_relation( // + log, // + sess, // + XR_REFERENCE_SPACE_TYPE_VIEW, // + baseSpc->type, // + viewLocateInfo->displayTime, // + &pure_relation); // // @todo the fov information that we get from xdev->hmd->views[i].fov is // not properly filled out in oh_device.c, fix before wasting time @@ -359,17 +381,18 @@ oxr_session_views(struct oxr_logger *log, viewState->viewStateFlags = 0; + //! @todo Do not hardcode IPD. + const struct xrt_vec3 eye_relation = { + sess->ipd_meters, + 0.0f, + 0.0f, + }; + for (uint32_t i = 0; i < num_views; i++) { - //! @todo Do not hardcode IPD. - struct xrt_vec3 eye_relation = { - sess->ipd_meters, - 0.0f, - 0.0f, - }; - struct xrt_pose view_pose; + struct xrt_pose view_pose = XRT_POSE_IDENTITY; // Get the per view pose from the device. - xdev->get_view_pose(xdev, &eye_relation, i, &view_pose); + xrt_device_get_view_pose(xdev, &eye_relation, i, &view_pose); // Do the magical space relation dance here. struct xrt_space_relation result = {0}; @@ -494,6 +517,10 @@ oxr_session_frame_wait(struct oxr_logger *log, struct oxr_session *sess, XrFrame ts_ms(sess), ns_to_ms(predicted_display_time), ns_to_ms(predicted_display_period)); } + if (sess->frame_timing_wait_sleep_ms > 0) { + os_nanosleep(U_TIME_1MS_IN_NS * sess->frame_timing_wait_sleep_ms); + } + return oxr_session_success_result(sess); } @@ -547,1378 +574,6 @@ oxr_session_frame_begin(struct oxr_logger *log, struct oxr_session *sess) return ret; } -static enum xrt_blend_mode -oxr_blend_mode_to_xrt(XrEnvironmentBlendMode blend_mode) -{ - switch (blend_mode) { - case XR_ENVIRONMENT_BLEND_MODE_OPAQUE: return XRT_BLEND_MODE_OPAQUE; - case XR_ENVIRONMENT_BLEND_MODE_ADDITIVE: return XRT_BLEND_MODE_ADDITIVE; - case XR_ENVIRONMENT_BLEND_MODE_ALPHA_BLEND: return XRT_BLEND_MODE_ALPHA_BLEND; - default: return (enum xrt_blend_mode)0; - } -} - -static XrResult -verify_space(struct oxr_logger *log, uint32_t layer_index, XrSpace space) -{ - if (space == XR_NULL_HANDLE) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->space == " - "XR_NULL_HANDLE) XrSpace must not be XR_NULL_HANDLE", - layer_index); - } - - return XR_SUCCESS; -} - -static XrResult -is_rect_neg(const XrRect2Di *imageRect) -{ - if (imageRect->offset.x < 0 || imageRect->offset.y < 0) { - return true; - } - - return false; -} - -static XrResult -is_rect_out_of_bounds(const XrRect2Di *imageRect, struct oxr_swapchain *sc) -{ - uint32_t total_width = imageRect->offset.x + imageRect->extent.width; - if (total_width > sc->width) { - return true; - } - uint32_t total_height = imageRect->offset.y + imageRect->extent.height; - if (total_height > sc->height) { - return true; - } - - return false; -} - -static XrResult -verify_quad_layer(struct xrt_compositor *xc, - struct oxr_logger *log, - uint32_t layer_index, - XrCompositionLayerQuad *quad, - struct xrt_device *head, - uint64_t timestamp) -{ - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, quad->subImage.swapchain); - - if (sc == NULL) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->subImage." - "swapchain) swapchain is NULL!", - layer_index); - } - - XrResult ret = verify_space(log, layer_index, quad->space); - if (ret != XR_SUCCESS) { - return ret; - } - - if (!math_quat_validate_within_1_percent((struct xrt_quat *)&quad->pose.orientation)) { - XrQuaternionf *q = &quad->pose.orientation; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->pose.orientation " - "== {%f %f %f %f}) is not a valid quat", - layer_index, q->x, q->y, q->z, q->w); - } - - if (!math_vec3_validate((struct xrt_vec3 *)&quad->pose.position)) { - XrVector3f *p = &quad->pose.position; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->pose.position " - "== {%f %f %f}) is not valid", - layer_index, p->x, p->y, p->z); - } - - if (sc->num_array_layers <= quad->subImage.imageArrayIndex) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->subImage.imageArrayIndex == " - "%u) Invalid swapchain array " - "index for quad layer (%u).", - layer_index, quad->subImage.imageArrayIndex, sc->num_array_layers); - } - - if (!sc->released.yes) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->subImage." - "swapchain) swapchain has not been released!", - layer_index); - } - - if (sc->released.index >= (int)sc->swapchain->num_images) { - return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, - "(frameEndInfo->layers[%u]->subImage.swapchain) internal " - "image index out of bounds", - layer_index); - } - - if (is_rect_neg(&quad->subImage.imageRect)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->subImage.imageRect.offset == " - "{%i, %i}) has negative component(s)", - layer_index, quad->subImage.imageRect.offset.x, quad->subImage.imageRect.offset.y); - } - - if (is_rect_out_of_bounds(&quad->subImage.imageRect, sc)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->subImage.imageRect == {{%i, " - "%i}, {%u, %u}}) imageRect out of image bounds (%u, %u)", - layer_index, quad->subImage.imageRect.offset.x, quad->subImage.imageRect.offset.y, - quad->subImage.imageRect.extent.width, quad->subImage.imageRect.extent.height, - sc->width, sc->height); - } - - return XR_SUCCESS; -} - -static XrResult -verify_depth_layer(struct xrt_compositor *xc, - struct oxr_logger *log, - uint32_t layer_index, - uint32_t i, - const XrCompositionLayerDepthInfoKHR *depth) -{ - if (depth->subImage.swapchain == XR_NULL_HANDLE) { - return oxr_error(log, XR_ERROR_HANDLE_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->next<" - "XrCompositionLayerDepthInfoKHR>.subImage." - "swapchain) is XR_NULL_HANDLE", - layer_index, i); - } - - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, depth->subImage.swapchain); - - if (!sc->released.yes) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->next<" - "XrCompositionLayerDepthInfoKHR>.subImage." - "swapchain) swapchain has not been released", - layer_index, i); - } - - if (sc->released.index >= (int)sc->swapchain->num_images) { - return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, - "(frameEndInfo->layers[%u]->views[%i]->next<" - "XrCompositionLayerDepthInfoKHR>.subImage." - "swapchain) internal image index out of bounds", - layer_index, i); - } - - if (sc->num_array_layers <= depth->subImage.imageArrayIndex) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->views[%i]->next<" - "XrCompositionLayerDepthInfoKHR>.subImage." - "imageArrayIndex == %u) Invalid swapchain array " - "index for projection layer (%u).", - layer_index, i, depth->subImage.imageArrayIndex, sc->num_array_layers); - } - - if (is_rect_neg(&depth->subImage.imageRect)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->next<" - "XrCompositionLayerDepthInfoKHR>.subImage." - "imageRect.offset == {%i, " - "%i}) has negative component(s)", - layer_index, i, depth->subImage.imageRect.offset.x, - depth->subImage.imageRect.offset.y); - } - - if (is_rect_out_of_bounds(&depth->subImage.imageRect, sc)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->next<" - "XrCompositionLayerDepthInfoKHR>.subImage." - "imageRect == {{%i, %i}, {%u, %u}}) imageRect out " - "of image bounds (%u, %u)", - layer_index, i, depth->subImage.imageRect.offset.x, depth->subImage.imageRect.offset.y, - depth->subImage.imageRect.extent.width, depth->subImage.imageRect.extent.height, - sc->width, sc->height); - } - - if (depth->minDepth < 0.0f || depth->minDepth > 1.0f) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->next<" - "XrCompositionLayerDepthInfoKHR>.minDepth) %f " - "must be in [0.0,1.0]", - layer_index, i, depth->minDepth); - } - - if (depth->maxDepth < 0.0f || depth->maxDepth > 1.0f) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->next<" - "XrCompositionLayerDepthInfoKHR>.maxDepth) %f " - "must be in [0.0,1.0]", - layer_index, i, depth->maxDepth); - } - - if (depth->minDepth > depth->maxDepth) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->next<" - "XrCompositionLayerDepthInfoKHR>.minDepth) %f " - "must be <= maxDepth %f ", - layer_index, i, depth->minDepth, depth->maxDepth); - } - - if (depth->nearZ == depth->farZ) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->next<" - "XrCompositionLayerDepthInfoKHR>.nearZ) %f " - "must be != farZ %f ", - layer_index, i, depth->nearZ, depth->farZ); - } - - - return XR_SUCCESS; -} - -static XrResult -verify_projection_layer(struct xrt_compositor *xc, - struct oxr_logger *log, - uint32_t layer_index, - XrCompositionLayerProjection *proj, - struct xrt_device *head, - uint64_t timestamp) -{ - XrResult ret = verify_space(log, layer_index, proj->space); - if (ret != XR_SUCCESS) { - return ret; - } - - if (proj->viewCount != 2) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->viewCount == %u) must be 2 for " - "projection layers and the current view configuration", - layer_index, proj->viewCount); - } - - // number of depth layers must be 0 or proj->viewCount - uint32_t num_depth_layers = 0; - - // Check for valid swapchain states. - for (uint32_t i = 0; i < proj->viewCount; i++) { - const XrCompositionLayerProjectionView *view = &proj->views[i]; - - //! @todo More validation? - if (!math_quat_validate_within_1_percent((struct xrt_quat *)&view->pose.orientation)) { - const XrQuaternionf *q = &view->pose.orientation; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->pose." - "orientation == {%f %f %f %f}) is not a valid quat", - layer_index, i, q->x, q->y, q->z, q->w); - } - - if (!math_vec3_validate((struct xrt_vec3 *)&view->pose.position)) { - const XrVector3f *p = &view->pose.position; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->pose." - "position == {%f %f %f}) is not valid", - layer_index, i, p->x, p->y, p->z); - } - - if (view->subImage.swapchain == XR_NULL_HANDLE) { - return oxr_error(log, XR_ERROR_HANDLE_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->subImage." - "swapchain is XR_NULL_HANDLE", - layer_index, i); - } - - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, view->subImage.swapchain); - - if (!sc->released.yes) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->views[%i].subImage." - "swapchain) swapchain has not been released", - layer_index, i); - } - - if (sc->released.index >= (int)sc->swapchain->num_images) { - return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, - "(frameEndInfo->layers[%u]->views[%i].subImage." - "swapchain) internal image index out of bounds", - layer_index, i); - } - - if (sc->num_array_layers <= view->subImage.imageArrayIndex) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->views[%i]->subImage." - "imageArrayIndex == %u) Invalid swapchain array " - "index for projection layer (%u).", - layer_index, i, view->subImage.imageArrayIndex, sc->num_array_layers); - } - - if (is_rect_neg(&view->subImage.imageRect)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->views[%i]-" - ">subImage.imageRect.offset == {%i, " - "%i}) has negative component(s)", - layer_index, i, view->subImage.imageRect.offset.x, - view->subImage.imageRect.offset.y); - } - - if (is_rect_out_of_bounds(&view->subImage.imageRect, sc)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->views[%i]->subImage." - "imageRect == {{%i, %i}, {%u, %u}}) imageRect out " - "of image bounds (%u, %u)", - layer_index, i, view->subImage.imageRect.offset.x, - view->subImage.imageRect.offset.y, view->subImage.imageRect.extent.width, - view->subImage.imageRect.extent.height, sc->width, sc->height); - } - -#ifdef XRT_FEATURE_OPENXR_LAYER_DEPTH - const XrCompositionLayerDepthInfoKHR *depth_info = OXR_GET_INPUT_FROM_CHAIN( - view, XR_TYPE_COMPOSITION_LAYER_DEPTH_INFO_KHR, XrCompositionLayerDepthInfoKHR); - - if (depth_info) { - ret = verify_depth_layer(xc, log, layer_index, i, depth_info); - if (ret != XR_SUCCESS) { - return ret; - } - num_depth_layers++; - } -#endif // XRT_FEATURE_OPENXR_LAYER_DEPTH - } - -#ifdef XRT_FEATURE_OPENXR_LAYER_DEPTH - if (num_depth_layers > 0 && num_depth_layers != proj->viewCount) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u] projection layer must have %u " - "depth layers or none, but has: %u)", - layer_index, proj->viewCount, num_depth_layers); - } -#endif // XRT_FEATURE_OPENXR_LAYER_DEPTH - - return XR_SUCCESS; -} - -static XrResult -verify_cube_layer(struct xrt_compositor *xc, - struct oxr_logger *log, - uint32_t layer_index, - const XrCompositionLayerCubeKHR *cube, - struct xrt_device *head, - uint64_t timestamp) -{ -#ifndef XRT_FEATURE_OPENXR_LAYER_CUBE - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->type) layer type " - "XrCompositionLayerCubeKHR not supported", - layer_index); -#else - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, cube->swapchain); - - if (sc == NULL) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->subImage." - "swapchain) swapchain is NULL!", - layer_index); - } - - XrResult ret = verify_space(log, layer_index, cube->space); - if (ret != XR_SUCCESS) { - return ret; - } - - if (!math_quat_validate_within_1_percent((struct xrt_quat *)&cube->orientation)) { - const XrQuaternionf *q = &cube->orientation; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->pose.orientation " - "== {%f %f %f %f}) is not a valid quat", - layer_index, q->x, q->y, q->z, q->w); - } - - if (sc->num_array_layers <= cube->imageArrayIndex) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->imageArrayIndex == %u) Invalid " - "swapchain array index for cube layer (%u).", - layer_index, cube->imageArrayIndex, sc->num_array_layers); - } - - if (!sc->released.yes) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->swapchain) " - "swapchain has not been released!", - layer_index); - } - - if (sc->released.index >= (int)sc->swapchain->num_images) { - return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, - "(frameEndInfo->layers[%u]->subImage.swapchain) internal " - "image index out of bounds", - layer_index); - } - - return XR_SUCCESS; -#endif -} - -static XrResult -verify_cylinder_layer(struct xrt_compositor *xc, - struct oxr_logger *log, - uint32_t layer_index, - const XrCompositionLayerCylinderKHR *cylinder, - struct xrt_device *head, - uint64_t timestamp) -{ -#ifndef XRT_FEATURE_OPENXR_LAYER_CYLINDER - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->type) layer type " - "XrCompositionLayerCylinderKHR not supported", - layer_index); -#else - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, cylinder->subImage.swapchain); - - if (sc == NULL) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->subImage." - "swapchain) swapchain is NULL!", - layer_index); - } - - XrResult ret = verify_space(log, layer_index, cylinder->space); - if (ret != XR_SUCCESS) { - return ret; - } - - if (!math_quat_validate_within_1_percent((struct xrt_quat *)&cylinder->pose.orientation)) { - const XrQuaternionf *q = &cylinder->pose.orientation; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->pose.orientation " - "== {%f %f %f %f}) is not a valid quat", - layer_index, q->x, q->y, q->z, q->w); - } - - if (!math_vec3_validate((struct xrt_vec3 *)&cylinder->pose.position)) { - const XrVector3f *p = &cylinder->pose.position; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->pose.position == " - "{%f %f %f}) is not valid", - layer_index, p->x, p->y, p->z); - } - - if (sc->num_array_layers <= cylinder->subImage.imageArrayIndex) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->subImage." - "imageArrayIndex == %u) Invalid swapchain " - "array index for cylinder layer (%u).", - layer_index, cylinder->subImage.imageArrayIndex, sc->num_array_layers); - } - - if (!sc->released.yes) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->subImage." - "swapchain) swapchain has not been released!", - layer_index); - } - - if (sc->released.index >= (int)sc->swapchain->num_images) { - return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, - "(frameEndInfo->layers[%u]->subImage.swapchain) internal " - "image index out of bounds", - layer_index); - } - - if (is_rect_neg(&cylinder->subImage.imageRect)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->subImage.imageRect.offset == " - "{%i, %i}) has negative component(s)", - layer_index, cylinder->subImage.imageRect.offset.x, - cylinder->subImage.imageRect.offset.y); - } - - if (is_rect_out_of_bounds(&cylinder->subImage.imageRect, sc)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->subImage.imageRect == {{%i, " - "%i}, {%u, %u}}) imageRect out of image bounds (%u, %u)", - layer_index, cylinder->subImage.imageRect.offset.x, - cylinder->subImage.imageRect.offset.y, cylinder->subImage.imageRect.extent.width, - cylinder->subImage.imageRect.extent.height, sc->width, sc->height); - } - - if (cylinder->radius < 0.f) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->radius == %f) " - "radius can not be negative", - layer_index, cylinder->radius); - } - - if (cylinder->centralAngle < 0.f || cylinder->centralAngle > (M_PI * 2)) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->centralAngle == " - "%f) centralAngle out of bounds", - layer_index, cylinder->centralAngle); - } - - if (cylinder->aspectRatio <= 0.f) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->aspectRatio == " - "%f) aspectRatio out of bounds", - layer_index, cylinder->aspectRatio); - } - - return XR_SUCCESS; -#endif -} - -static XrResult -verify_equirect1_layer(struct xrt_compositor *xc, - struct oxr_logger *log, - uint32_t layer_index, - const XrCompositionLayerEquirectKHR *equirect, - struct xrt_device *head, - uint64_t timestamp) -{ -#ifndef XRT_FEATURE_OPENXR_LAYER_EQUIRECT1 - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->type) layer type " - "XrCompositionLayerEquirectKHR not supported", - layer_index); -#else - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, equirect->subImage.swapchain); - - if (sc == NULL) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->subImage." - "swapchain) swapchain is NULL!", - layer_index); - } - - XrResult ret = verify_space(log, layer_index, equirect->space); - if (ret != XR_SUCCESS) { - return ret; - } - - if (!math_quat_validate_within_1_percent((struct xrt_quat *)&equirect->pose.orientation)) { - const XrQuaternionf *q = &equirect->pose.orientation; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->pose.orientation " - "== {%f %f %f %f}) is not a valid quat", - layer_index, q->x, q->y, q->z, q->w); - } - - if (!math_vec3_validate((struct xrt_vec3 *)&equirect->pose.position)) { - const XrVector3f *p = &equirect->pose.position; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->pose.position == " - "{%f %f %f}) is not valid", - layer_index, p->x, p->y, p->z); - } - - if (sc->num_array_layers <= equirect->subImage.imageArrayIndex) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->subImage." - "imageArrayIndex == %u) Invalid swapchain " - "array index for equirect layer (%u).", - layer_index, equirect->subImage.imageArrayIndex, sc->num_array_layers); - } - - if (!sc->released.yes) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->subImage." - "swapchain) swapchain has not been released!", - layer_index); - } - - if (sc->released.index >= (int)sc->swapchain->num_images) { - return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, - "(frameEndInfo->layers[%u]->subImage.swapchain) internal " - "image index out of bounds", - layer_index); - } - - if (is_rect_neg(&equirect->subImage.imageRect)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->subImage.imageRect.offset == " - "{%i, %i}) has negative component(s)", - layer_index, equirect->subImage.imageRect.offset.x, - equirect->subImage.imageRect.offset.y); - } - - if (is_rect_out_of_bounds(&equirect->subImage.imageRect, sc)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->subImage.imageRect == {{%i, " - "%i}, {%u, %u}}) imageRect out of image bounds (%u, %u)", - layer_index, equirect->subImage.imageRect.offset.x, - equirect->subImage.imageRect.offset.y, equirect->subImage.imageRect.extent.width, - equirect->subImage.imageRect.extent.height, sc->width, sc->height); - } - - if (equirect->radius < .0f) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->radius == %f) " - "radius out of bounds", - layer_index, equirect->radius); - } - - return XR_SUCCESS; -#endif -} - -static XrResult -verify_equirect2_layer(struct xrt_compositor *xc, - struct oxr_logger *log, - uint32_t layer_index, - const XrCompositionLayerEquirect2KHR *equirect, - struct xrt_device *head, - uint64_t timestamp) -{ -#ifndef XRT_FEATURE_OPENXR_LAYER_EQUIRECT2 - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->type) layer type " - "XrCompositionLayerEquirect2KHR not supported", - layer_index); -#else - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, equirect->subImage.swapchain); - - if (sc == NULL) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->subImage." - "swapchain) swapchain is NULL!", - layer_index); - } - - XrResult ret = verify_space(log, layer_index, equirect->space); - if (ret != XR_SUCCESS) { - return ret; - } - - if (!math_quat_validate_within_1_percent((struct xrt_quat *)&equirect->pose.orientation)) { - const XrQuaternionf *q = &equirect->pose.orientation; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->pose.orientation " - "== {%f %f %f %f}) is not a valid quat", - layer_index, q->x, q->y, q->z, q->w); - } - - if (!math_vec3_validate((struct xrt_vec3 *)&equirect->pose.position)) { - const XrVector3f *p = &equirect->pose.position; - return oxr_error(log, XR_ERROR_POSE_INVALID, - "(frameEndInfo->layers[%u]->pose.position == " - "{%f %f %f}) is not valid", - layer_index, p->x, p->y, p->z); - } - - if (sc->num_array_layers <= equirect->subImage.imageArrayIndex) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->subImage." - "imageArrayIndex == %u) Invalid swapchain " - "array index for equirect layer (%u).", - layer_index, equirect->subImage.imageArrayIndex, sc->num_array_layers); - } - - if (!sc->released.yes) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->subImage." - "swapchain) swapchain has not been released!", - layer_index); - } - - if (sc->released.index >= (int)sc->swapchain->num_images) { - return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, - "(frameEndInfo->layers[%u]->subImage.swapchain) internal " - "image index out of bounds", - layer_index); - } - - if (is_rect_neg(&equirect->subImage.imageRect)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->subImage.imageRect.offset == " - "{%i, %i}) has negative component(s)", - layer_index, equirect->subImage.imageRect.offset.x, - equirect->subImage.imageRect.offset.y); - } - - if (is_rect_out_of_bounds(&equirect->subImage.imageRect, sc)) { - return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, - "(frameEndInfo->layers[%u]->subImage.imageRect == {{%i, " - "%i}, {%u, %u}}) imageRect out of image bounds (%u, %u)", - layer_index, equirect->subImage.imageRect.offset.x, - equirect->subImage.imageRect.offset.y, equirect->subImage.imageRect.extent.width, - equirect->subImage.imageRect.extent.height, sc->width, sc->height); - } - - if (equirect->centralHorizontalAngle < .0f) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->layers[%u]->centralHorizontalAngle == %f) " - "centralHorizontalAngle out of bounds", - layer_index, equirect->centralHorizontalAngle); - } - - /* - * Accept all angle ranges here, since we are dealing with π - * and we don't want floating point errors to prevent the client - * to display the full sphere. - */ - - return XR_SUCCESS; -#endif -} - -static enum xrt_layer_composition_flags -convert_layer_flags(XrSwapchainUsageFlags xr_flags) -{ - enum xrt_layer_composition_flags flags = 0; - - if ((xr_flags & XR_COMPOSITION_LAYER_CORRECT_CHROMATIC_ABERRATION_BIT) != 0) { - flags |= XRT_LAYER_COMPOSITION_CORRECT_CHROMATIC_ABERRATION_BIT; - } - if ((xr_flags & XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT) != 0) { - flags |= XRT_LAYER_COMPOSITION_BLEND_TEXTURE_SOURCE_ALPHA_BIT; - } - if ((xr_flags & XR_COMPOSITION_LAYER_UNPREMULTIPLIED_ALPHA_BIT) != 0) { - flags |= XRT_LAYER_COMPOSITION_UNPREMULTIPLIED_ALPHA_BIT; - } - - return flags; -} - -static enum xrt_layer_eye_visibility -convert_eye_visibility(XrSwapchainUsageFlags xr_visibility) -{ - enum xrt_layer_eye_visibility visibility = 0; - - if (xr_visibility == XR_EYE_VISIBILITY_BOTH) { - visibility = XRT_LAYER_EYE_VISIBILITY_BOTH; - } - if (xr_visibility == XR_EYE_VISIBILITY_LEFT) { - visibility = XRT_LAYER_EYE_VISIBILITY_LEFT_BIT; - } - if (xr_visibility == XR_EYE_VISIBILITY_RIGHT) { - visibility = XRT_LAYER_EYE_VISIBILITY_RIGHT_BIT; - } - - return visibility; -} - -static bool -handle_space(struct oxr_logger *log, - struct oxr_session *sess, - struct oxr_space *spc, - const struct xrt_pose *pose_ptr, - const struct xrt_pose *inv_offset, - uint64_t timestamp, - struct xrt_pose *out_pose) -{ - struct xrt_pose pose = *pose_ptr; - - // The pose might be valid for OpenXR, but not good enough for math. - if (!math_quat_validate(&pose.orientation)) { - math_quat_normalize(&pose.orientation); - } - - if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { - // The space might have a pose, transform that in as well. - math_pose_transform(&spc->pose, &pose, &pose); - } else if (spc->is_reference) { - // The space might have a pose, transform that in as well. - math_pose_transform(&spc->pose, &pose, &pose); - - // Remove the tracking system origin offset. - math_pose_transform(inv_offset, &pose, &pose); - - if (spc->type == XR_REFERENCE_SPACE_TYPE_LOCAL) { - if (!initial_head_relation_valid(sess)) { - return false; - } - math_pose_transform(&sess->initial_head_relation.pose, &pose, &pose); - } - - } else { - //! @todo Action space handling not very complete - - struct oxr_action_input *input = NULL; - - oxr_action_get_pose_input(log, sess, spc->act_key, &spc->subaction_paths, &input); - - // If the input isn't active. - if (input == NULL) { - //! @todo just don't render the quad here? - return false; - } - - - struct xrt_space_relation out_relation; - - oxr_xdev_get_space_relation(log, sess->sys->inst, input->xdev, input->input->name, timestamp, - &out_relation); - - struct xrt_pose device_pose = out_relation.pose; - - // The space might have a pose, transform that in as well. - math_pose_transform(&spc->pose, &device_pose, &device_pose); - - math_pose_transform(&device_pose, &pose, &pose); - - // Remove the tracking system origin offset. - math_pose_transform(inv_offset, &pose, &pose); - } - - - *out_pose = pose; - - return true; -} - -static XrResult -submit_quad_layer(struct oxr_session *sess, - struct xrt_compositor *xc, - struct oxr_logger *log, - XrCompositionLayerQuad *quad, - struct xrt_device *head, - struct xrt_pose *inv_offset, - uint64_t timestamp) -{ - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, quad->subImage.swapchain); - struct oxr_space *spc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, quad->space); - - enum xrt_layer_composition_flags flags = convert_layer_flags(quad->layerFlags); - - struct xrt_pose *pose_ptr = (struct xrt_pose *)&quad->pose; - - struct xrt_pose pose; - if (!handle_space(log, sess, spc, pose_ptr, inv_offset, timestamp, &pose)) { - return XR_SUCCESS; - } - - if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { - flags |= XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT; - } - - struct xrt_layer_data data; - U_ZERO(&data); - data.type = XRT_LAYER_QUAD; - data.name = XRT_INPUT_GENERIC_HEAD_POSE; - data.timestamp = timestamp; - data.flags = flags; - - struct xrt_vec2 *size = (struct xrt_vec2 *)&quad->size; - struct xrt_rect *rect = (struct xrt_rect *)&quad->subImage.imageRect; - - data.quad.visibility = convert_eye_visibility(quad->eyeVisibility); - data.quad.sub.image_index = sc->released.index; - data.quad.sub.array_index = quad->subImage.imageArrayIndex; - data.quad.sub.rect = *rect; - data.quad.pose = pose; - data.quad.size = *size; - - CALL_CHK(xrt_comp_layer_quad(xc, head, sc->swapchain, &data)); - - return XR_SUCCESS; -} - -static XrResult -submit_projection_layer(struct xrt_compositor *xc, - struct oxr_logger *log, - XrCompositionLayerProjection *proj, - struct xrt_device *head, - struct xrt_pose *inv_offset, - uint64_t timestamp) -{ - struct oxr_space *spc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, proj->space); - struct oxr_swapchain *d_scs[2] = {NULL, NULL}; - struct oxr_swapchain *scs[2]; - struct xrt_pose *pose_ptr[2]; - struct xrt_pose pose[2]; - - enum xrt_layer_composition_flags flags = convert_layer_flags(proj->layerFlags); - - uint32_t num_chains = ARRAY_SIZE(scs); - for (uint32_t i = 0; i < num_chains; i++) { - scs[i] = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, proj->views[i].subImage.swapchain); - pose_ptr[i] = (struct xrt_pose *)&proj->views[i].pose; - pose[i] = *pose_ptr[i]; - - // The pose might be valid for OpenXR, but not good enough for math. - if (!math_quat_validate(&pose[i].orientation)) { - math_quat_normalize(&pose[i].orientation); - } - } - - if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { - flags |= XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT; - // The space might have a pose, transform that in as well. - math_pose_transform(&spc->pose, &pose[0], &pose[0]); - math_pose_transform(&spc->pose, &pose[1], &pose[1]); - } else { - //! @todo Handle action spaces. - - // The space might have a pose, transform that in as well. - math_pose_transform(&spc->pose, &pose[0], &pose[0]); - math_pose_transform(&spc->pose, &pose[1], &pose[1]); - - // Remove the tracking system origin offset. - math_pose_transform(inv_offset, &pose[0], &pose[0]); - math_pose_transform(inv_offset, &pose[1], &pose[1]); - } - - struct xrt_rect *l_rect = (struct xrt_rect *)&proj->views[0].subImage.imageRect; - struct xrt_fov *l_fov = (struct xrt_fov *)&proj->views[0].fov; - struct xrt_rect *r_rect = (struct xrt_rect *)&proj->views[1].subImage.imageRect; - struct xrt_fov *r_fov = (struct xrt_fov *)&proj->views[1].fov; - - struct xrt_layer_data data; - U_ZERO(&data); - data.type = XRT_LAYER_STEREO_PROJECTION; - data.name = XRT_INPUT_GENERIC_HEAD_POSE; - data.timestamp = timestamp; - data.flags = flags; - - data.stereo.l.sub.image_index = scs[0]->released.index; - data.stereo.l.sub.array_index = proj->views[0].subImage.imageArrayIndex; - data.stereo.l.sub.rect = *l_rect; - data.stereo.l.fov = *l_fov; - data.stereo.l.pose = pose[0]; - - data.stereo.r.sub.image_index = scs[1]->released.index; - data.stereo.r.sub.array_index = proj->views[1].subImage.imageArrayIndex; - data.stereo.r.sub.rect = *r_rect; - data.stereo.r.fov = *r_fov; - data.stereo.r.pose = pose[1]; - - - -#ifdef XRT_FEATURE_OPENXR_LAYER_DEPTH - const XrCompositionLayerDepthInfoKHR *d_l = OXR_GET_INPUT_FROM_CHAIN( - &proj->views[0], XR_TYPE_COMPOSITION_LAYER_DEPTH_INFO_KHR, XrCompositionLayerDepthInfoKHR); - if (d_l) { - data.stereo_depth.l_d.far_z = d_l->farZ; - data.stereo_depth.l_d.near_z = d_l->nearZ; - data.stereo_depth.l_d.max_depth = d_l->maxDepth; - data.stereo_depth.l_d.min_depth = d_l->minDepth; - - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, d_l->subImage.swapchain); - - struct xrt_rect *d_l_rect = (struct xrt_rect *)&d_l->subImage.imageRect; - data.stereo_depth.l_d.sub.image_index = sc->released.index; - data.stereo_depth.l_d.sub.array_index = d_l->subImage.imageArrayIndex; - data.stereo_depth.l_d.sub.rect = *d_l_rect; - - // Need to pass this in. - d_scs[0] = sc; - } - - const XrCompositionLayerDepthInfoKHR *d_r = OXR_GET_INPUT_FROM_CHAIN( - &proj->views[1], XR_TYPE_COMPOSITION_LAYER_DEPTH_INFO_KHR, XrCompositionLayerDepthInfoKHR); - - if (d_r) { - data.stereo_depth.r_d.far_z = d_r->farZ; - data.stereo_depth.r_d.near_z = d_r->nearZ; - data.stereo_depth.r_d.max_depth = d_r->maxDepth; - data.stereo_depth.r_d.min_depth = d_r->minDepth; - - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, d_r->subImage.swapchain); - - struct xrt_rect *d_l_rect = (struct xrt_rect *)&d_r->subImage.imageRect; - data.stereo_depth.r_d.sub.image_index = sc->released.index; - data.stereo_depth.r_d.sub.array_index = d_r->subImage.imageArrayIndex; - data.stereo_depth.r_d.sub.rect = *d_l_rect; - - // Need to pass this in. - d_scs[1] = sc; - } -#endif // XRT_FEATURE_OPENXR_LAYER_DEPTH - - if (d_scs[0] != NULL && d_scs[1] != NULL) { -#ifdef XRT_FEATURE_OPENXR_LAYER_DEPTH - data.type = XRT_LAYER_STEREO_PROJECTION_DEPTH; - CALL_CHK(xrt_comp_layer_stereo_projection_depth(xc, head, - scs[0]->swapchain, // Left - scs[1]->swapchain, // Right - d_scs[0]->swapchain, // Left - d_scs[1]->swapchain, // Right - &data)); -#else - assert(false && "Should not get here"); -#endif // XRT_FEATURE_OPENXR_LAYER_DEPTH - } else { - CALL_CHK(xrt_comp_layer_stereo_projection(xc, head, - scs[0]->swapchain, // Left - scs[1]->swapchain, // Right - &data)); - } - - return XR_SUCCESS; -} - -static void -submit_cube_layer(struct oxr_session *sess, - struct xrt_compositor *xc, - struct oxr_logger *log, - const XrCompositionLayerCubeKHR *cube, - struct xrt_device *head, - struct xrt_pose *inv_offset, - uint64_t timestamp) -{ - // Not implemented -} - -static XrResult -submit_cylinder_layer(struct oxr_session *sess, - struct xrt_compositor *xc, - struct oxr_logger *log, - const XrCompositionLayerCylinderKHR *cylinder, - struct xrt_device *head, - struct xrt_pose *inv_offset, - uint64_t timestamp) -{ - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, cylinder->subImage.swapchain); - struct oxr_space *spc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, cylinder->space); - - enum xrt_layer_composition_flags flags = convert_layer_flags(cylinder->layerFlags); - enum xrt_layer_eye_visibility visibility = convert_eye_visibility(cylinder->eyeVisibility); - - struct xrt_pose *pose_ptr = (struct xrt_pose *)&cylinder->pose; - - struct xrt_pose pose; - if (!handle_space(log, sess, spc, pose_ptr, inv_offset, timestamp, &pose)) { - return XR_SUCCESS; - } - - if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { - flags |= XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT; - } - - struct xrt_layer_data data; - U_ZERO(&data); - data.type = XRT_LAYER_CYLINDER; - data.name = XRT_INPUT_GENERIC_HEAD_POSE; - data.timestamp = timestamp; - data.flags = flags; - - struct xrt_rect *rect = (struct xrt_rect *)&cylinder->subImage.imageRect; - - data.cylinder.visibility = visibility; - data.cylinder.sub.image_index = sc->released.index; - data.cylinder.sub.array_index = cylinder->subImage.imageArrayIndex; - data.cylinder.sub.rect = *rect; - data.cylinder.pose = pose; - data.cylinder.radius = cylinder->radius; - data.cylinder.central_angle = cylinder->centralAngle; - data.cylinder.aspect_ratio = cylinder->aspectRatio; - - CALL_CHK(xrt_comp_layer_cylinder(xc, head, sc->swapchain, &data)); - - return XR_SUCCESS; -} - -static XrResult -submit_equirect1_layer(struct oxr_session *sess, - struct xrt_compositor *xc, - struct oxr_logger *log, - const XrCompositionLayerEquirectKHR *equirect, - struct xrt_device *head, - struct xrt_pose *inv_offset, - uint64_t timestamp) -{ - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, equirect->subImage.swapchain); - struct oxr_space *spc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, equirect->space); - - enum xrt_layer_composition_flags flags = convert_layer_flags(equirect->layerFlags); - - struct xrt_pose *pose_ptr = (struct xrt_pose *)&equirect->pose; - - struct xrt_pose pose; - if (!handle_space(log, sess, spc, pose_ptr, inv_offset, timestamp, &pose)) { - return XR_SUCCESS; - } - - if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { - flags |= XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT; - } - - struct xrt_layer_data data; - U_ZERO(&data); - data.type = XRT_LAYER_EQUIRECT1; - data.name = XRT_INPUT_GENERIC_HEAD_POSE; - data.timestamp = timestamp; - data.flags = flags; - - struct xrt_rect *rect = (struct xrt_rect *)&equirect->subImage.imageRect; - - data.equirect1.visibility = convert_eye_visibility(equirect->eyeVisibility); - data.equirect1.sub.image_index = sc->released.index; - data.equirect1.sub.array_index = equirect->subImage.imageArrayIndex; - data.equirect1.sub.rect = *rect; - data.equirect1.pose = pose; - - data.equirect1.radius = equirect->radius; - - struct xrt_vec2 *scale = (struct xrt_vec2 *)&equirect->scale; - struct xrt_vec2 *bias = (struct xrt_vec2 *)&equirect->bias; - - data.equirect1.scale = *scale; - data.equirect1.bias = *bias; - - CALL_CHK(xrt_comp_layer_equirect1(xc, head, sc->swapchain, &data)); - - return XR_SUCCESS; -} - -static void -do_synchronize_state_change(struct oxr_logger *log, struct oxr_session *sess) -{ - if (!sess->has_ended_once && sess->state < XR_SESSION_STATE_VISIBLE) { - oxr_session_change_state(log, sess, XR_SESSION_STATE_SYNCHRONIZED); - sess->has_ended_once = true; - } -} - -static XrResult -submit_equirect2_layer(struct oxr_session *sess, - struct xrt_compositor *xc, - struct oxr_logger *log, - const XrCompositionLayerEquirect2KHR *equirect, - struct xrt_device *head, - struct xrt_pose *inv_offset, - uint64_t timestamp) -{ - struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, equirect->subImage.swapchain); - struct oxr_space *spc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, equirect->space); - - enum xrt_layer_composition_flags flags = convert_layer_flags(equirect->layerFlags); - - struct xrt_pose *pose_ptr = (struct xrt_pose *)&equirect->pose; - - struct xrt_pose pose; - if (!handle_space(log, sess, spc, pose_ptr, inv_offset, timestamp, &pose)) { - return XR_SUCCESS; - } - - if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { - flags |= XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT; - } - - struct xrt_layer_data data; - U_ZERO(&data); - data.type = XRT_LAYER_EQUIRECT2; - data.name = XRT_INPUT_GENERIC_HEAD_POSE; - data.timestamp = timestamp; - data.flags = flags; - - struct xrt_rect *rect = (struct xrt_rect *)&equirect->subImage.imageRect; - - data.equirect2.visibility = convert_eye_visibility(equirect->eyeVisibility); - data.equirect2.sub.image_index = sc->released.index; - data.equirect2.sub.array_index = equirect->subImage.imageArrayIndex; - data.equirect2.sub.rect = *rect; - data.equirect2.pose = pose; - - data.equirect2.radius = equirect->radius; - data.equirect2.central_horizontal_angle = equirect->centralHorizontalAngle; - data.equirect2.upper_vertical_angle = equirect->upperVerticalAngle; - data.equirect2.lower_vertical_angle = equirect->lowerVerticalAngle; - - CALL_CHK(xrt_comp_layer_equirect2(xc, head, sc->swapchain, &data)); - - return XR_SUCCESS; -} - -XrResult -oxr_session_frame_end(struct oxr_logger *log, struct oxr_session *sess, const XrFrameEndInfo *frameEndInfo) -{ - /* - * Session state and call order. - */ - - if (!is_running(sess)) { - return oxr_error(log, XR_ERROR_SESSION_NOT_RUNNING, "Session is not running"); - } - if (!sess->frame_started) { - return oxr_error(log, XR_ERROR_CALL_ORDER_INVALID, "Frame not begun with xrBeginFrame"); - } - - if (frameEndInfo->displayTime <= 0) { - return oxr_error(log, XR_ERROR_TIME_INVALID, - "(frameEndInfo->displayTime == %" PRIi64 - ") zero or a negative value is not a valid XrTime", - frameEndInfo->displayTime); - } - - int64_t timestamp = time_state_ts_to_monotonic_ns(sess->sys->inst->timekeeping, frameEndInfo->displayTime); - if (sess->frame_timing_spew) { - oxr_log(log, "End frame at %8.3fms with display time %8.3fms", ts_ms(sess), ns_to_ms(timestamp)); - } - - struct xrt_compositor *xc = sess->compositor; - - /* - * early out for headless sessions. - */ - if (xc == NULL) { - sess->frame_started = false; - - os_mutex_lock(&sess->active_wait_frames_lock); - sess->active_wait_frames--; - os_mutex_unlock(&sess->active_wait_frames_lock); - - do_synchronize_state_change(log, sess); - - return oxr_session_success_result(sess); - } - - - /* - * Blend mode. - * XR_ERROR_ENVIRONMENT_BLEND_MODE_UNSUPPORTED must always be reported, - * even with 0 layers. - */ - - enum xrt_blend_mode blend_mode = oxr_blend_mode_to_xrt(frameEndInfo->environmentBlendMode); - - if (blend_mode == 0) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, - "(frameEndInfo->environmentBlendMode == " - "0x%08x) unknown environment blend mode", - frameEndInfo->environmentBlendMode); - } - - struct xrt_device *xdev = GET_XDEV_BY_ROLE(sess->sys, head); - if ((blend_mode & xdev->hmd->blend_mode) == 0) { - //! @todo Make integer print to string. - return oxr_error(log, XR_ERROR_ENVIRONMENT_BLEND_MODE_UNSUPPORTED, - "(frameEndInfo->environmentBlendMode == %u) " - "is not supported", - frameEndInfo->environmentBlendMode); - } - - /* - * Early out for discarded frame if layer count is 0. - */ - if (frameEndInfo->layerCount == 0) { - - os_mutex_lock(&sess->active_wait_frames_lock); - sess->active_wait_frames--; - os_mutex_unlock(&sess->active_wait_frames_lock); - - CALL_CHK(xrt_comp_discard_frame(xc, sess->frame_id.begun)); - sess->frame_id.begun = -1; - sess->frame_started = false; - - do_synchronize_state_change(log, sess); - - return oxr_session_success_result(sess); - } - - - /* - * Layers. - */ - - if (frameEndInfo->layers == NULL) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, "(frameEndInfo->layers == NULL)"); - } - - for (uint32_t i = 0; i < frameEndInfo->layerCount; i++) { - const XrCompositionLayerBaseHeader *layer = frameEndInfo->layers[i]; - if (layer == NULL) { - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u] == NULL) " - "layer can not be null", - i); - } - - XrResult res; - - switch (layer->type) { - case XR_TYPE_COMPOSITION_LAYER_PROJECTION: - res = verify_projection_layer(xc, log, i, (XrCompositionLayerProjection *)layer, xdev, - frameEndInfo->displayTime); - break; - case XR_TYPE_COMPOSITION_LAYER_QUAD: - res = verify_quad_layer(xc, log, i, (XrCompositionLayerQuad *)layer, xdev, - frameEndInfo->displayTime); - break; - case XR_TYPE_COMPOSITION_LAYER_CUBE_KHR: - res = verify_cube_layer(xc, log, i, (XrCompositionLayerCubeKHR *)layer, xdev, - frameEndInfo->displayTime); - break; - case XR_TYPE_COMPOSITION_LAYER_CYLINDER_KHR: - res = verify_cylinder_layer(xc, log, i, (XrCompositionLayerCylinderKHR *)layer, xdev, - frameEndInfo->displayTime); - break; - case XR_TYPE_COMPOSITION_LAYER_EQUIRECT_KHR: - res = verify_equirect1_layer(xc, log, i, (XrCompositionLayerEquirectKHR *)layer, xdev, - frameEndInfo->displayTime); - break; - case XR_TYPE_COMPOSITION_LAYER_EQUIRECT2_KHR: - res = verify_equirect2_layer(xc, log, i, (XrCompositionLayerEquirect2KHR *)layer, xdev, - frameEndInfo->displayTime); - break; - default: - return oxr_error(log, XR_ERROR_LAYER_INVALID, - "(frameEndInfo->layers[%u]->type) " - "layer type not supported", - i); - } - - if (res != XR_SUCCESS) { - return res; - } - } - - - /* - * Done verifying. - */ - - // Do state change if needed. - do_synchronize_state_change(log, sess); - - struct xrt_pose inv_offset = {0}; - math_pose_invert(&xdev->tracking_origin->offset, &inv_offset); - - CALL_CHK(xrt_comp_layer_begin(xc, sess->frame_id.begun, blend_mode)); - - for (uint32_t i = 0; i < frameEndInfo->layerCount; i++) { - const XrCompositionLayerBaseHeader *layer = frameEndInfo->layers[i]; - assert(layer != NULL); - - int64_t timestamp = - time_state_ts_to_monotonic_ns(sess->sys->inst->timekeeping, frameEndInfo->displayTime); - - switch (layer->type) { - case XR_TYPE_COMPOSITION_LAYER_PROJECTION: - submit_projection_layer(xc, log, (XrCompositionLayerProjection *)layer, xdev, &inv_offset, - timestamp); - break; - case XR_TYPE_COMPOSITION_LAYER_QUAD: - submit_quad_layer(sess, xc, log, (XrCompositionLayerQuad *)layer, xdev, &inv_offset, timestamp); - break; - case XR_TYPE_COMPOSITION_LAYER_CUBE_KHR: - submit_cube_layer(sess, xc, log, (XrCompositionLayerCubeKHR *)layer, xdev, &inv_offset, - timestamp); - break; - case XR_TYPE_COMPOSITION_LAYER_CYLINDER_KHR: - submit_cylinder_layer(sess, xc, log, (XrCompositionLayerCylinderKHR *)layer, xdev, &inv_offset, - timestamp); - break; - case XR_TYPE_COMPOSITION_LAYER_EQUIRECT_KHR: - submit_equirect1_layer(sess, xc, log, (XrCompositionLayerEquirectKHR *)layer, xdev, &inv_offset, - timestamp); - break; - case XR_TYPE_COMPOSITION_LAYER_EQUIRECT2_KHR: - submit_equirect2_layer(sess, xc, log, (XrCompositionLayerEquirect2KHR *)layer, xdev, - &inv_offset, timestamp); - break; - default: assert(false && "invalid layer type"); - } - } - - CALL_CHK(xrt_comp_layer_commit(xc, sess->frame_id.begun, XRT_GRAPHICS_SYNC_HANDLE_INVALID)); - sess->frame_id.begun = -1; - - sess->frame_started = false; - - os_mutex_lock(&sess->active_wait_frames_lock); - sess->active_wait_frames--; - os_mutex_unlock(&sess->active_wait_frames_lock); - - return oxr_session_success_result(sess); -} - static XrResult oxr_session_destroy(struct oxr_logger *log, struct oxr_handle_base *hb) { @@ -1961,6 +616,10 @@ oxr_session_destroy(struct oxr_logger *log, struct oxr_handle_base *hb) return oxr_error((LOG), XR_ERROR_RUNTIME_FAILURE, "Failed to create native compositor! '%i'", \ xret); \ } \ + if ((SESS)->sys->xsysc->xmcc != NULL) { \ + xrt_syscomp_set_state((SESS)->sys->xsysc, &(SESS)->xcn->base, true, true); \ + xrt_syscomp_set_z_order((SESS)->sys->xsysc, &(SESS)->xcn->base, 0); \ + } \ } while (false) #define OXR_SESSION_ALLOCATE(LOG, SYS, OUT) \ @@ -2017,12 +676,33 @@ oxr_session_create_impl(struct oxr_logger *log, XrGraphicsBindingVulkanKHR const *vulkan = OXR_GET_INPUT_FROM_CHAIN(createInfo, XR_TYPE_GRAPHICS_BINDING_VULKAN_KHR, XrGraphicsBindingVulkanKHR); if (vulkan != NULL) { + OXR_VERIFY_ARG_NOT_ZERO(log, vulkan->instance); + OXR_VERIFY_ARG_NOT_ZERO(log, vulkan->physicalDevice); + if (vulkan->device == VK_NULL_HANDLE) { + return oxr_error(log, XR_ERROR_GRAPHICS_DEVICE_INVALID, "VkDevice must not be VK_NULL_HANDLE"); + } + if (!sys->gotten_requirements) { return oxr_error(log, XR_ERROR_GRAPHICS_REQUIREMENTS_CALL_MISSING, "Has not called " "xrGetVulkanGraphicsRequirementsKHR"); } + if (sys->suggested_vulkan_physical_device == VK_NULL_HANDLE) { + char *fn = sys->inst->extensions.KHR_vulkan_enable ? "xrGetVulkanGraphicsDeviceKHR" + : "xrGetVulkanGraphicsDevice2KHR"; + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "Has not called %s", fn); + } + + if (sys->suggested_vulkan_physical_device != vulkan->physicalDevice) { + char *fn = sys->inst->extensions.KHR_vulkan_enable ? "xrGetVulkanGraphicsDeviceKHR" + : "xrGetVulkanGraphicsDevice2KHR"; + return oxr_error( + log, XR_ERROR_VALIDATION_FAILURE, + "XrGraphicsBindingVulkanKHR::physicalDevice %p must match device %p specified by %s", + (void *)vulkan->physicalDevice, (void *)sys->suggested_vulkan_physical_device, fn); + } + OXR_SESSION_ALLOCATE(log, sys, *out_session); OXR_ALLOCATE_NATIVE_COMPOSITOR(log, xsi, *out_session); return oxr_session_populate_vk(log, sys, vulkan, *out_session); @@ -2101,6 +781,7 @@ oxr_session_create(struct oxr_logger *log, sess->ipd_meters = debug_get_num_option_ipd() / 1000.0f; sess->frame_timing_spew = debug_get_bool_option_frame_timing_spew(); + sess->frame_timing_wait_sleep_ms = debug_get_num_option_wait_frame_sleep(); oxr_session_change_state(log, sess, XR_SESSION_STATE_IDLE); oxr_session_change_state(log, sess, XR_SESSION_STATE_READY); @@ -2139,6 +820,11 @@ oxr_session_hand_joints(struct oxr_logger *log, XrHandJointVelocitiesEXT *vel = OXR_GET_OUTPUT_FROM_CHAIN(locations, XR_TYPE_HAND_JOINT_VELOCITIES_EXT, XrHandJointVelocitiesEXT); + if (hand_tracker->xdev == NULL) { + locations->isActive = false; + return XR_SUCCESS; + } + struct xrt_device *xdev = hand_tracker->xdev; enum xrt_input_name name = hand_tracker->input_name; @@ -2246,7 +932,14 @@ oxr_session_hand_joints(struct oxr_logger *log, } } - locations->isActive = true; + if (value.is_active) { + locations->isActive = true; + } else { + locations->isActive = false; + for (uint32_t i = 0; i < locations->jointCount; i++) { + locations->jointLocations[i].locationFlags = XRT_SPACE_RELATION_BITMASK_NONE; + } + } return XR_SUCCESS; } diff --git a/src/xrt/state_trackers/oxr/oxr_session_frame_end.c b/src/xrt/state_trackers/oxr/oxr_session_frame_end.c new file mode 100644 index 000000000..52eb5c5c4 --- /dev/null +++ b/src/xrt/state_trackers/oxr/oxr_session_frame_end.c @@ -0,0 +1,1367 @@ +// Copyright 2018-2021, Collabora, Ltd. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Holds session end frame functions. + * @author Jakob Bornecrantz + * @author Moses Turner + * @ingroup oxr_main + */ + +#include "xrt/xrt_device.h" +#include "xrt/xrt_config_build.h" +#include "xrt/xrt_config_have.h" + +#include "os/os_time.h" + +#include "util/u_debug.h" +#include "util/u_misc.h" +#include "util/u_time.h" +#include "util/u_verify.h" + +#include "math/m_api.h" +#include "math/m_mathinclude.h" +#include "math/m_space.h" + +#include "oxr_objects.h" +#include "oxr_logger.h" +#include "oxr_two_call.h" +#include "oxr_handle.h" +#include "oxr_chain.h" +#include "oxr_api_verify.h" +#include "oxr_chain.h" + +#include +#include +#include +#include + + +/* + * + * Helper functions and defines. + * + */ + +#define CALL_CHK(call) \ + if ((call) == XRT_ERROR_IPC_FAILURE) { \ + return oxr_error(log, XR_ERROR_INSTANCE_LOST, "Error in function call over IPC"); \ + } + +static double +ns_to_ms(int64_t ns) +{ + double ms = ((double)ns) * 1. / 1000. * 1. / 1000.; + return ms; +} + +static double +ts_ms(struct oxr_session *sess) +{ + timepoint_ns now = time_state_get_now(sess->sys->inst->timekeeping); + int64_t monotonic = time_state_ts_to_monotonic_ns(sess->sys->inst->timekeeping, now); + return ns_to_ms(monotonic); +} + +static bool +is_session_running(struct oxr_session *sess) +{ + return sess->has_begun; +} + +static XrResult +is_rect_neg(const XrRect2Di *imageRect) +{ + if (imageRect->offset.x < 0 || imageRect->offset.y < 0) { + return true; + } + + return false; +} + +static XrResult +is_rect_out_of_bounds(const XrRect2Di *imageRect, struct oxr_swapchain *sc) +{ + uint32_t total_width = imageRect->offset.x + imageRect->extent.width; + if (total_width > sc->width) { + return true; + } + uint32_t total_height = imageRect->offset.y + imageRect->extent.height; + if (total_height > sc->height) { + return true; + } + + return false; +} + +static enum xrt_blend_mode +convert_blend_mode(XrEnvironmentBlendMode blend_mode) +{ + switch (blend_mode) { + case XR_ENVIRONMENT_BLEND_MODE_OPAQUE: return XRT_BLEND_MODE_OPAQUE; + case XR_ENVIRONMENT_BLEND_MODE_ADDITIVE: return XRT_BLEND_MODE_ADDITIVE; + case XR_ENVIRONMENT_BLEND_MODE_ALPHA_BLEND: return XRT_BLEND_MODE_ALPHA_BLEND; + default: return (enum xrt_blend_mode)0; + } +} + +static enum xrt_layer_composition_flags +convert_layer_flags(XrSwapchainUsageFlags xr_flags) +{ + enum xrt_layer_composition_flags flags = 0; + + if ((xr_flags & XR_COMPOSITION_LAYER_CORRECT_CHROMATIC_ABERRATION_BIT) != 0) { + flags |= XRT_LAYER_COMPOSITION_CORRECT_CHROMATIC_ABERRATION_BIT; + } + if ((xr_flags & XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT) != 0) { + flags |= XRT_LAYER_COMPOSITION_BLEND_TEXTURE_SOURCE_ALPHA_BIT; + } + if ((xr_flags & XR_COMPOSITION_LAYER_UNPREMULTIPLIED_ALPHA_BIT) != 0) { + flags |= XRT_LAYER_COMPOSITION_UNPREMULTIPLIED_ALPHA_BIT; + } + + return flags; +} + +static enum xrt_layer_eye_visibility +convert_eye_visibility(XrSwapchainUsageFlags xr_visibility) +{ + enum xrt_layer_eye_visibility visibility = 0; + + if (xr_visibility == XR_EYE_VISIBILITY_BOTH) { + visibility = XRT_LAYER_EYE_VISIBILITY_BOTH; + } + if (xr_visibility == XR_EYE_VISIBILITY_LEFT) { + visibility = XRT_LAYER_EYE_VISIBILITY_LEFT_BIT; + } + if (xr_visibility == XR_EYE_VISIBILITY_RIGHT) { + visibility = XRT_LAYER_EYE_VISIBILITY_RIGHT_BIT; + } + + return visibility; +} + +static void +fill_in_sub_image(const struct oxr_swapchain *sc, const XrSwapchainSubImage *oxr_sub, struct xrt_sub_image *xsub) +{ + const struct xrt_rect *rect = (const struct xrt_rect *)&oxr_sub->imageRect; + + xsub->image_index = sc->released.index; + xsub->array_index = oxr_sub->imageArrayIndex; + xsub->rect = *rect; + xsub->norm_rect.w = rect->extent.w / (double)sc->width; + xsub->norm_rect.h = rect->extent.h / (double)sc->height; + xsub->norm_rect.x = rect->offset.w / (double)sc->width; + xsub->norm_rect.y = rect->offset.h / (double)sc->height; +} + + +/* + * + * Verify functions. + * + */ + +static XrResult +verify_space(struct oxr_logger *log, uint32_t layer_index, XrSpace space) +{ + if (space == XR_NULL_HANDLE) { + return oxr_error( + log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->space == XR_NULL_HANDLE) XrSpace must not be XR_NULL_HANDLE", + layer_index); + } + + return XR_SUCCESS; +} + +static XrResult +verify_quad_layer(struct xrt_compositor *xc, + struct oxr_logger *log, + uint32_t layer_index, + XrCompositionLayerQuad *quad, + struct xrt_device *head, + uint64_t timestamp) +{ + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, quad->subImage.swapchain); + + if (sc == NULL) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->subImage.swapchain) swapchain is NULL!", layer_index); + } + + XrResult ret = verify_space(log, layer_index, quad->space); + if (ret != XR_SUCCESS) { + return ret; + } + + if (!math_quat_validate_within_1_percent((struct xrt_quat *)&quad->pose.orientation)) { + XrQuaternionf *q = &quad->pose.orientation; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->pose.orientation == {%f %f %f %f}) is not a valid quat", + layer_index, q->x, q->y, q->z, q->w); + } + + if (!math_vec3_validate((struct xrt_vec3 *)&quad->pose.position)) { + XrVector3f *p = &quad->pose.position; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->pose.position == {%f %f %f}) is not valid", layer_index, + p->x, p->y, p->z); + } + + if (sc->num_array_layers <= quad->subImage.imageArrayIndex) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->subImage.imageArrayIndex == %u) Invalid swapchain array " + "index for quad layer (%u).", + layer_index, quad->subImage.imageArrayIndex, sc->num_array_layers); + } + + if (!sc->released.yes) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->subImage.swapchain) swapchain has not been released!", + layer_index); + } + + if (sc->released.index >= (int)sc->swapchain->num_images) { + return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, + "(frameEndInfo->layers[%u]->subImage.swapchain) internal image index out of bounds", + layer_index); + } + + if (is_rect_neg(&quad->subImage.imageRect)) { + return oxr_error( + log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->subImage.imageRect.offset == {%i, %i}) has negative component(s)", + layer_index, quad->subImage.imageRect.offset.x, quad->subImage.imageRect.offset.y); + } + + if (is_rect_out_of_bounds(&quad->subImage.imageRect, sc)) { + return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->subImage.imageRect == {{%i, %i}, {%u, %u}}) imageRect out " + "of image bounds (%u, %u)", + layer_index, quad->subImage.imageRect.offset.x, quad->subImage.imageRect.offset.y, + quad->subImage.imageRect.extent.width, quad->subImage.imageRect.extent.height, + sc->width, sc->height); + } + + return XR_SUCCESS; +} + +static XrResult +verify_depth_layer(struct xrt_compositor *xc, + struct oxr_logger *log, + uint32_t layer_index, + uint32_t i, + const XrCompositionLayerDepthInfoKHR *depth) +{ + if (depth->subImage.swapchain == XR_NULL_HANDLE) { + return oxr_error(log, XR_ERROR_HANDLE_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->next.subImage." + "swapchain) is XR_NULL_HANDLE", + layer_index, i); + } + + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, depth->subImage.swapchain); + + if (!sc->released.yes) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->next.subImage." + "swapchain) swapchain has not been released", + layer_index, i); + } + + if (sc->released.index >= (int)sc->swapchain->num_images) { + return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, + "(frameEndInfo->layers[%u]->views[%i]->next.subImage." + "swapchain) internal image index out of bounds", + layer_index, i); + } + + if (sc->num_array_layers <= depth->subImage.imageArrayIndex) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->views[%i]->next.subImage." + "imageArrayIndex == %u) Invalid swapchain array index for projection layer (%u).", + layer_index, i, depth->subImage.imageArrayIndex, sc->num_array_layers); + } + + if (is_rect_neg(&depth->subImage.imageRect)) { + return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->next.subImage." + "imageRect.offset == {%i, %i}) has negative component(s)", + layer_index, i, depth->subImage.imageRect.offset.x, + depth->subImage.imageRect.offset.y); + } + + if (is_rect_out_of_bounds(&depth->subImage.imageRect, sc)) { + return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->next.subImage." + "imageRect == {{%i, %i}, {%u, %u}}) imageRect out of image bounds (%u, %u)", + layer_index, i, depth->subImage.imageRect.offset.x, depth->subImage.imageRect.offset.y, + depth->subImage.imageRect.extent.width, depth->subImage.imageRect.extent.height, + sc->width, sc->height); + } + + if (depth->minDepth < 0.0f || depth->minDepth > 1.0f) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->next.minDepth) " + "%f must be in [0.0,1.0]", + layer_index, i, depth->minDepth); + } + + if (depth->maxDepth < 0.0f || depth->maxDepth > 1.0f) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->next.maxDepth) " + "%f must be in [0.0,1.0]", + layer_index, i, depth->maxDepth); + } + + if (depth->minDepth > depth->maxDepth) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->next.minDepth) " + "%f must be <= maxDepth %f ", + layer_index, i, depth->minDepth, depth->maxDepth); + } + + if (depth->nearZ == depth->farZ) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->next.nearZ) %f " + "must be != farZ %f ", + layer_index, i, depth->nearZ, depth->farZ); + } + + + return XR_SUCCESS; +} + +static XrResult +verify_projection_layer(struct xrt_compositor *xc, + struct oxr_logger *log, + uint32_t layer_index, + XrCompositionLayerProjection *proj, + struct xrt_device *head, + uint64_t timestamp) +{ + XrResult ret = verify_space(log, layer_index, proj->space); + if (ret != XR_SUCCESS) { + return ret; + } + + if (proj->viewCount != 2) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->viewCount == %u) must be 2 for projection layers and the " + "current view configuration", + layer_index, proj->viewCount); + } + + // number of depth layers must be 0 or proj->viewCount + uint32_t num_depth_layers = 0; + + // Check for valid swapchain states. + for (uint32_t i = 0; i < proj->viewCount; i++) { + const XrCompositionLayerProjectionView *view = &proj->views[i]; + + //! @todo More validation? + if (!math_quat_validate_within_1_percent((struct xrt_quat *)&view->pose.orientation)) { + const XrQuaternionf *q = &view->pose.orientation; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->pose." + "orientation == {%f %f %f %f}) is not a valid quat", + layer_index, i, q->x, q->y, q->z, q->w); + } + + if (!math_vec3_validate((struct xrt_vec3 *)&view->pose.position)) { + const XrVector3f *p = &view->pose.position; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->pose." + "position == {%f %f %f}) is not valid", + layer_index, i, p->x, p->y, p->z); + } + + if (view->subImage.swapchain == XR_NULL_HANDLE) { + return oxr_error(log, XR_ERROR_HANDLE_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->subImage." + "swapchain is XR_NULL_HANDLE", + layer_index, i); + } + + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, view->subImage.swapchain); + + if (!sc->released.yes) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->views[%i].subImage." + "swapchain) swapchain has not been released", + layer_index, i); + } + + if (sc->released.index >= (int)sc->swapchain->num_images) { + return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, + "(frameEndInfo->layers[%u]->views[%i].subImage." + "swapchain) internal image index out of bounds", + layer_index, i); + } + + if (sc->num_array_layers <= view->subImage.imageArrayIndex) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->views[%i]->subImage." + "imageArrayIndex == %u) Invalid swapchain array " + "index for projection layer (%u).", + layer_index, i, view->subImage.imageArrayIndex, sc->num_array_layers); + } + + if (is_rect_neg(&view->subImage.imageRect)) { + return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->views[%i]-" + ">subImage.imageRect.offset == {%i, " + "%i}) has negative component(s)", + layer_index, i, view->subImage.imageRect.offset.x, + view->subImage.imageRect.offset.y); + } + + if (is_rect_out_of_bounds(&view->subImage.imageRect, sc)) { + return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->views[%i]->subImage." + "imageRect == {{%i, %i}, {%u, %u}}) imageRect out " + "of image bounds (%u, %u)", + layer_index, i, view->subImage.imageRect.offset.x, + view->subImage.imageRect.offset.y, view->subImage.imageRect.extent.width, + view->subImage.imageRect.extent.height, sc->width, sc->height); + } + +#ifdef XRT_FEATURE_OPENXR_LAYER_DEPTH + const XrCompositionLayerDepthInfoKHR *depth_info = OXR_GET_INPUT_FROM_CHAIN( + view, XR_TYPE_COMPOSITION_LAYER_DEPTH_INFO_KHR, XrCompositionLayerDepthInfoKHR); + + if (depth_info) { + ret = verify_depth_layer(xc, log, layer_index, i, depth_info); + if (ret != XR_SUCCESS) { + return ret; + } + num_depth_layers++; + } +#endif // XRT_FEATURE_OPENXR_LAYER_DEPTH + } + +#ifdef XRT_FEATURE_OPENXR_LAYER_DEPTH + if (num_depth_layers > 0 && num_depth_layers != proj->viewCount) { + return oxr_error( + log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u] projection layer must have %u depth layers or none, but has: %u)", + layer_index, proj->viewCount, num_depth_layers); + } +#endif // XRT_FEATURE_OPENXR_LAYER_DEPTH + + return XR_SUCCESS; +} + +static XrResult +verify_cube_layer(struct xrt_compositor *xc, + struct oxr_logger *log, + uint32_t layer_index, + const XrCompositionLayerCubeKHR *cube, + struct xrt_device *head, + uint64_t timestamp) +{ +#ifndef XRT_FEATURE_OPENXR_LAYER_CUBE + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->type) layer type " + "XrCompositionLayerCubeKHR not supported", + layer_index); +#else + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, cube->swapchain); + + if (sc == NULL) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->subImage.swapchain) swapchain is NULL!", layer_index); + } + + XrResult ret = verify_space(log, layer_index, cube->space); + if (ret != XR_SUCCESS) { + return ret; + } + + if (!math_quat_validate_within_1_percent((struct xrt_quat *)&cube->orientation)) { + const XrQuaternionf *q = &cube->orientation; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->pose.orientation == {%f %f %f %f}) is not a valid quat", + layer_index, q->x, q->y, q->z, q->w); + } + + if (sc->num_array_layers <= cube->imageArrayIndex) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->imageArrayIndex == %u) Invalid swapchain array index for " + "cube layer (%u).", + layer_index, cube->imageArrayIndex, sc->num_array_layers); + } + + if (!sc->released.yes) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->swapchain) swapchain has not been released!", layer_index); + } + + if (sc->released.index >= (int)sc->swapchain->num_images) { + return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, + "(frameEndInfo->layers[%u]->subImage.swapchain) internal image index out of bounds", + layer_index); + } + + return XR_SUCCESS; +#endif +} + +static XrResult +verify_cylinder_layer(struct xrt_compositor *xc, + struct oxr_logger *log, + uint32_t layer_index, + const XrCompositionLayerCylinderKHR *cylinder, + struct xrt_device *head, + uint64_t timestamp) +{ +#ifndef XRT_FEATURE_OPENXR_LAYER_CYLINDER + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->type) layer type " + "XrCompositionLayerCylinderKHR not supported", + layer_index); +#else + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, cylinder->subImage.swapchain); + + if (sc == NULL) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->subImage.swapchain) swapchain is NULL!", layer_index); + } + + XrResult ret = verify_space(log, layer_index, cylinder->space); + if (ret != XR_SUCCESS) { + return ret; + } + + if (!math_quat_validate_within_1_percent((struct xrt_quat *)&cylinder->pose.orientation)) { + const XrQuaternionf *q = &cylinder->pose.orientation; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->pose.orientation == {%f %f %f %f}) is not a valid quat", + layer_index, q->x, q->y, q->z, q->w); + } + + if (!math_vec3_validate((struct xrt_vec3 *)&cylinder->pose.position)) { + const XrVector3f *p = &cylinder->pose.position; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->pose.position == {%f %f %f}) is not valid", layer_index, + p->x, p->y, p->z); + } + + if (sc->num_array_layers <= cylinder->subImage.imageArrayIndex) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->subImage.imageArrayIndex == %u) Invalid swapchain array " + "index for cylinder layer (%u).", + layer_index, cylinder->subImage.imageArrayIndex, sc->num_array_layers); + } + + if (!sc->released.yes) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->subImage.swapchain) swapchain has not been released!", + layer_index); + } + + if (sc->released.index >= (int)sc->swapchain->num_images) { + return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, + "(frameEndInfo->layers[%u]->subImage.swapchain) internal image index out of bounds", + layer_index); + } + + if (is_rect_neg(&cylinder->subImage.imageRect)) { + return oxr_error( + log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->subImage.imageRect.offset == {%i, %i}) has negative component(s)", + layer_index, cylinder->subImage.imageRect.offset.x, cylinder->subImage.imageRect.offset.y); + } + + if (is_rect_out_of_bounds(&cylinder->subImage.imageRect, sc)) { + return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->subImage.imageRect == {{%i, %i}, {%u, %u}}) imageRect out " + "of image bounds (%u, %u)", + layer_index, cylinder->subImage.imageRect.offset.x, + cylinder->subImage.imageRect.offset.y, cylinder->subImage.imageRect.extent.width, + cylinder->subImage.imageRect.extent.height, sc->width, sc->height); + } + + if (cylinder->radius < 0.f) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->radius == %f) radius can not be negative", layer_index, + cylinder->radius); + } + + if (cylinder->centralAngle < 0.f || cylinder->centralAngle > (M_PI * 2)) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->centralAngle == %f) centralAngle out of bounds", + layer_index, cylinder->centralAngle); + } + + if (cylinder->aspectRatio <= 0.f) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->aspectRatio == %f) aspectRatio out of bounds", layer_index, + cylinder->aspectRatio); + } + + return XR_SUCCESS; +#endif +} + +static XrResult +verify_equirect1_layer(struct xrt_compositor *xc, + struct oxr_logger *log, + uint32_t layer_index, + const XrCompositionLayerEquirectKHR *equirect, + struct xrt_device *head, + uint64_t timestamp) +{ +#ifndef XRT_FEATURE_OPENXR_LAYER_EQUIRECT1 + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->type) layer type " + "XrCompositionLayerEquirectKHR not supported", + layer_index); +#else + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, equirect->subImage.swapchain); + + if (sc == NULL) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->subImage.swapchain) swapchain is NULL!", layer_index); + } + + XrResult ret = verify_space(log, layer_index, equirect->space); + if (ret != XR_SUCCESS) { + return ret; + } + + if (!math_quat_validate_within_1_percent((struct xrt_quat *)&equirect->pose.orientation)) { + const XrQuaternionf *q = &equirect->pose.orientation; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->pose.orientation == {%f %f %f %f}) is not a valid quat", + layer_index, q->x, q->y, q->z, q->w); + } + + if (!math_vec3_validate((struct xrt_vec3 *)&equirect->pose.position)) { + const XrVector3f *p = &equirect->pose.position; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->pose.position == {%f %f %f}) is not valid", layer_index, + p->x, p->y, p->z); + } + + if (sc->num_array_layers <= equirect->subImage.imageArrayIndex) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->subImage.imageArrayIndex == %u) Invalid swapchain array " + "index for equirect layer (%u).", + layer_index, equirect->subImage.imageArrayIndex, sc->num_array_layers); + } + + if (!sc->released.yes) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->subImage.swapchain) swapchain has not been released!", + layer_index); + } + + if (sc->released.index >= (int)sc->swapchain->num_images) { + return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, + "(frameEndInfo->layers[%u]->subImage.swapchain) internal image index out of bounds", + layer_index); + } + + if (is_rect_neg(&equirect->subImage.imageRect)) { + return oxr_error( + log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->subImage.imageRect.offset == {%i, %i}) has negative component(s)", + layer_index, equirect->subImage.imageRect.offset.x, equirect->subImage.imageRect.offset.y); + } + + if (is_rect_out_of_bounds(&equirect->subImage.imageRect, sc)) { + return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->subImage.imageRect == {{%i, %i}, {%u, %u}}) imageRect out " + "of image bounds (%u, %u)", + layer_index, equirect->subImage.imageRect.offset.x, + equirect->subImage.imageRect.offset.y, equirect->subImage.imageRect.extent.width, + equirect->subImage.imageRect.extent.height, sc->width, sc->height); + } + + if (equirect->radius < .0f) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->radius == %f) radius out of bounds", layer_index, + equirect->radius); + } + + return XR_SUCCESS; +#endif +} + +static XrResult +verify_equirect2_layer(struct xrt_compositor *xc, + struct oxr_logger *log, + uint32_t layer_index, + const XrCompositionLayerEquirect2KHR *equirect, + struct xrt_device *head, + uint64_t timestamp) +{ +#ifndef XRT_FEATURE_OPENXR_LAYER_EQUIRECT2 + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->type) layer type XrCompositionLayerEquirect2KHR not supported", + layer_index); +#else + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, equirect->subImage.swapchain); + + if (sc == NULL) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->subImage.swapchain) swapchain is NULL!", layer_index); + } + + XrResult ret = verify_space(log, layer_index, equirect->space); + if (ret != XR_SUCCESS) { + return ret; + } + + if (!math_quat_validate_within_1_percent((struct xrt_quat *)&equirect->pose.orientation)) { + const XrQuaternionf *q = &equirect->pose.orientation; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->pose.orientation == {%f %f %f %f}) is not a valid quat", + layer_index, q->x, q->y, q->z, q->w); + } + + if (!math_vec3_validate((struct xrt_vec3 *)&equirect->pose.position)) { + const XrVector3f *p = &equirect->pose.position; + return oxr_error(log, XR_ERROR_POSE_INVALID, + "(frameEndInfo->layers[%u]->pose.position == {%f %f %f}) is not valid", layer_index, + p->x, p->y, p->z); + } + + if (sc->num_array_layers <= equirect->subImage.imageArrayIndex) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->subImage.imageArrayIndex == %u) Invalid swapchain array " + "index for equirect layer (%u).", + layer_index, equirect->subImage.imageArrayIndex, sc->num_array_layers); + } + + if (!sc->released.yes) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->subImage.swapchain) swapchain has not been released!", + layer_index); + } + + if (sc->released.index >= (int)sc->swapchain->num_images) { + return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, + "(frameEndInfo->layers[%u]->subImage.swapchain) internal image index out of bounds", + layer_index); + } + + if (is_rect_neg(&equirect->subImage.imageRect)) { + return oxr_error( + log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->subImage.imageRect.offset == {%i, %i}) has negative component(s)", + layer_index, equirect->subImage.imageRect.offset.x, equirect->subImage.imageRect.offset.y); + } + + if (is_rect_out_of_bounds(&equirect->subImage.imageRect, sc)) { + return oxr_error(log, XR_ERROR_SWAPCHAIN_RECT_INVALID, + "(frameEndInfo->layers[%u]->subImage.imageRect == {{%i, %i}, {%u, %u}}) imageRect out " + "of image bounds (%u, %u)", + layer_index, equirect->subImage.imageRect.offset.x, + equirect->subImage.imageRect.offset.y, equirect->subImage.imageRect.extent.width, + equirect->subImage.imageRect.extent.height, sc->width, sc->height); + } + + if (equirect->centralHorizontalAngle < .0f) { + return oxr_error( + log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->layers[%u]->centralHorizontalAngle == %f) centralHorizontalAngle out of bounds", + layer_index, equirect->centralHorizontalAngle); + } + + /* + * Accept all angle ranges here, since we are dealing with π + * and we don't want floating point errors to prevent the client + * to display the full sphere. + */ + + return XR_SUCCESS; +#endif +} + + +/* + * + * Submit functions. + * + */ + +static bool +handle_space(struct oxr_logger *log, + struct oxr_session *sess, + struct oxr_space *spc, + const struct xrt_pose *pose_ptr, + const struct xrt_pose *inv_offset, + uint64_t timestamp, + struct xrt_pose *out_pose) +{ + struct xrt_pose pose = *pose_ptr; + + // The pose might be valid for OpenXR, but not good enough for math. + if (!math_quat_validate(&pose.orientation)) { + math_quat_normalize(&pose.orientation); + } + + if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { + // The space might have a pose, transform that in as well. + math_pose_transform(&spc->pose, &pose, &pose); + } else if (spc->is_reference) { + // The space might have a pose, transform that in as well. + math_pose_transform(&spc->pose, &pose, &pose); + + // Remove the tracking system origin offset. + math_pose_transform(inv_offset, &pose, &pose); + + if (spc->type == XR_REFERENCE_SPACE_TYPE_LOCAL) { + if (!initial_head_relation_valid(sess)) { + return false; + } + math_pose_transform(&sess->initial_head_relation.pose, &pose, &pose); + } + + } else { + //! @todo Action space handling not very complete + + struct oxr_action_input *input = NULL; + + oxr_action_get_pose_input(log, sess, spc->act_key, &spc->subaction_paths, &input); + + // If the input isn't active. + if (input == NULL) { + //! @todo just don't render the quad here? + return false; + } + + + struct xrt_space_relation out_relation; + + oxr_xdev_get_space_relation(log, sess->sys->inst, input->xdev, input->input->name, timestamp, + &out_relation); + + struct xrt_pose device_pose = out_relation.pose; + + // The space might have a pose, transform that in as well. + math_pose_transform(&spc->pose, &device_pose, &device_pose); + + math_pose_transform(&device_pose, &pose, &pose); + + // Remove the tracking system origin offset. + math_pose_transform(inv_offset, &pose, &pose); + } + + + *out_pose = pose; + + return true; +} + +static XrResult +submit_quad_layer(struct oxr_session *sess, + struct xrt_compositor *xc, + struct oxr_logger *log, + XrCompositionLayerQuad *quad, + struct xrt_device *head, + struct xrt_pose *inv_offset, + uint64_t timestamp) +{ + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, quad->subImage.swapchain); + struct oxr_space *spc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, quad->space); + + enum xrt_layer_composition_flags flags = convert_layer_flags(quad->layerFlags); + + struct xrt_pose *pose_ptr = (struct xrt_pose *)&quad->pose; + + struct xrt_pose pose; + if (!handle_space(log, sess, spc, pose_ptr, inv_offset, timestamp, &pose)) { + return XR_SUCCESS; + } + + if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { + flags |= XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT; + } + + struct xrt_layer_data data; + U_ZERO(&data); + data.type = XRT_LAYER_QUAD; + data.name = XRT_INPUT_GENERIC_HEAD_POSE; + data.timestamp = timestamp; + data.flags = flags; + + struct xrt_vec2 *size = (struct xrt_vec2 *)&quad->size; + + data.quad.visibility = convert_eye_visibility(quad->eyeVisibility); + data.quad.pose = pose; + data.quad.size = *size; + fill_in_sub_image(sc, &quad->subImage, &data.quad.sub); + + CALL_CHK(xrt_comp_layer_quad(xc, head, sc->swapchain, &data)); + + return XR_SUCCESS; +} + +static XrResult +submit_projection_layer(struct oxr_session *sess, + struct xrt_compositor *xc, + struct oxr_logger *log, + XrCompositionLayerProjection *proj, + struct xrt_device *head, + struct xrt_pose *inv_offset, + uint64_t timestamp) +{ + struct oxr_space *spc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, proj->space); + struct oxr_swapchain *d_scs[2] = {NULL, NULL}; + struct oxr_swapchain *scs[2]; + struct xrt_pose *pose_ptr; + struct xrt_pose pose[2]; + + enum xrt_layer_composition_flags flags = convert_layer_flags(proj->layerFlags); + + uint32_t num_chains = ARRAY_SIZE(scs); + for (uint32_t i = 0; i < num_chains; i++) { + scs[i] = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, proj->views[i].subImage.swapchain); + pose_ptr = (struct xrt_pose *)&proj->views[i].pose; + + if (!handle_space(log, sess, spc, pose_ptr, inv_offset, timestamp, &pose[i])) { + return XR_SUCCESS; + } + } + + if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { + flags |= XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT; + } + + struct xrt_fov *l_fov = (struct xrt_fov *)&proj->views[0].fov; + struct xrt_fov *r_fov = (struct xrt_fov *)&proj->views[1].fov; + + struct xrt_layer_data data; + U_ZERO(&data); + data.type = XRT_LAYER_STEREO_PROJECTION; + data.name = XRT_INPUT_GENERIC_HEAD_POSE; + data.timestamp = timestamp; + data.flags = flags; + data.stereo.l.fov = *l_fov; + data.stereo.l.pose = pose[0]; + data.stereo.r.fov = *r_fov; + data.stereo.r.pose = pose[1]; + fill_in_sub_image(scs[0], &proj->views[0].subImage, &data.stereo.l.sub); + fill_in_sub_image(scs[1], &proj->views[1].subImage, &data.stereo.r.sub); + + +#ifdef XRT_FEATURE_OPENXR_LAYER_DEPTH + const XrCompositionLayerDepthInfoKHR *d_l = OXR_GET_INPUT_FROM_CHAIN( + &proj->views[0], XR_TYPE_COMPOSITION_LAYER_DEPTH_INFO_KHR, XrCompositionLayerDepthInfoKHR); + if (d_l) { + data.stereo_depth.l_d.far_z = d_l->farZ; + data.stereo_depth.l_d.near_z = d_l->nearZ; + data.stereo_depth.l_d.max_depth = d_l->maxDepth; + data.stereo_depth.l_d.min_depth = d_l->minDepth; + + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, d_l->subImage.swapchain); + + fill_in_sub_image(sc, &d_l->subImage, &data.stereo_depth.l_d.sub); + + // Need to pass this in. + d_scs[0] = sc; + } + + const XrCompositionLayerDepthInfoKHR *d_r = OXR_GET_INPUT_FROM_CHAIN( + &proj->views[1], XR_TYPE_COMPOSITION_LAYER_DEPTH_INFO_KHR, XrCompositionLayerDepthInfoKHR); + + if (d_r) { + data.stereo_depth.r_d.far_z = d_r->farZ; + data.stereo_depth.r_d.near_z = d_r->nearZ; + data.stereo_depth.r_d.max_depth = d_r->maxDepth; + data.stereo_depth.r_d.min_depth = d_r->minDepth; + + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, d_r->subImage.swapchain); + + fill_in_sub_image(sc, &d_r->subImage, &data.stereo_depth.r_d.sub); + + // Need to pass this in. + d_scs[1] = sc; + } +#endif // XRT_FEATURE_OPENXR_LAYER_DEPTH + + if (d_scs[0] != NULL && d_scs[1] != NULL) { +#ifdef XRT_FEATURE_OPENXR_LAYER_DEPTH + data.type = XRT_LAYER_STEREO_PROJECTION_DEPTH; + CALL_CHK(xrt_comp_layer_stereo_projection_depth(xc, head, + scs[0]->swapchain, // Left + scs[1]->swapchain, // Right + d_scs[0]->swapchain, // Left + d_scs[1]->swapchain, // Right + &data)); +#else + assert(false && "Should not get here"); +#endif // XRT_FEATURE_OPENXR_LAYER_DEPTH + } else { + CALL_CHK(xrt_comp_layer_stereo_projection(xc, head, + scs[0]->swapchain, // Left + scs[1]->swapchain, // Right + &data)); + } + + return XR_SUCCESS; +} + +static void +submit_cube_layer(struct oxr_session *sess, + struct xrt_compositor *xc, + struct oxr_logger *log, + const XrCompositionLayerCubeKHR *cube, + struct xrt_device *head, + struct xrt_pose *inv_offset, + uint64_t timestamp) +{ + // Not implemented +} + +static XrResult +submit_cylinder_layer(struct oxr_session *sess, + struct xrt_compositor *xc, + struct oxr_logger *log, + const XrCompositionLayerCylinderKHR *cylinder, + struct xrt_device *head, + struct xrt_pose *inv_offset, + uint64_t timestamp) +{ + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, cylinder->subImage.swapchain); + struct oxr_space *spc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, cylinder->space); + + enum xrt_layer_composition_flags flags = convert_layer_flags(cylinder->layerFlags); + enum xrt_layer_eye_visibility visibility = convert_eye_visibility(cylinder->eyeVisibility); + + struct xrt_pose *pose_ptr = (struct xrt_pose *)&cylinder->pose; + + struct xrt_pose pose; + if (!handle_space(log, sess, spc, pose_ptr, inv_offset, timestamp, &pose)) { + return XR_SUCCESS; + } + + if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { + flags |= XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT; + } + + struct xrt_layer_data data; + U_ZERO(&data); + data.type = XRT_LAYER_CYLINDER; + data.name = XRT_INPUT_GENERIC_HEAD_POSE; + data.timestamp = timestamp; + data.flags = flags; + + data.cylinder.visibility = visibility; + data.cylinder.pose = pose; + data.cylinder.radius = cylinder->radius; + data.cylinder.central_angle = cylinder->centralAngle; + data.cylinder.aspect_ratio = cylinder->aspectRatio; + fill_in_sub_image(sc, &cylinder->subImage, &data.cylinder.sub); + + CALL_CHK(xrt_comp_layer_cylinder(xc, head, sc->swapchain, &data)); + + return XR_SUCCESS; +} + +static XrResult +submit_equirect1_layer(struct oxr_session *sess, + struct xrt_compositor *xc, + struct oxr_logger *log, + const XrCompositionLayerEquirectKHR *equirect, + struct xrt_device *head, + struct xrt_pose *inv_offset, + uint64_t timestamp) +{ + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, equirect->subImage.swapchain); + struct oxr_space *spc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, equirect->space); + + enum xrt_layer_composition_flags flags = convert_layer_flags(equirect->layerFlags); + + struct xrt_pose *pose_ptr = (struct xrt_pose *)&equirect->pose; + + struct xrt_pose pose; + if (!handle_space(log, sess, spc, pose_ptr, inv_offset, timestamp, &pose)) { + return XR_SUCCESS; + } + + if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { + flags |= XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT; + } + + struct xrt_layer_data data; + U_ZERO(&data); + data.type = XRT_LAYER_EQUIRECT1; + data.name = XRT_INPUT_GENERIC_HEAD_POSE; + data.timestamp = timestamp; + data.flags = flags; + data.equirect1.visibility = convert_eye_visibility(equirect->eyeVisibility); + data.equirect1.pose = pose; + data.equirect1.radius = equirect->radius; + fill_in_sub_image(sc, &equirect->subImage, &data.equirect1.sub); + + + struct xrt_vec2 *scale = (struct xrt_vec2 *)&equirect->scale; + struct xrt_vec2 *bias = (struct xrt_vec2 *)&equirect->bias; + + data.equirect1.scale = *scale; + data.equirect1.bias = *bias; + + CALL_CHK(xrt_comp_layer_equirect1(xc, head, sc->swapchain, &data)); + + return XR_SUCCESS; +} + +static void +do_synchronize_state_change(struct oxr_logger *log, struct oxr_session *sess) +{ + if (!sess->has_ended_once && sess->state < XR_SESSION_STATE_VISIBLE) { + oxr_session_change_state(log, sess, XR_SESSION_STATE_SYNCHRONIZED); + sess->has_ended_once = true; + } +} + +static XrResult +submit_equirect2_layer(struct oxr_session *sess, + struct xrt_compositor *xc, + struct oxr_logger *log, + const XrCompositionLayerEquirect2KHR *equirect, + struct xrt_device *head, + struct xrt_pose *inv_offset, + uint64_t timestamp) +{ + struct oxr_swapchain *sc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_swapchain *, equirect->subImage.swapchain); + struct oxr_space *spc = XRT_CAST_OXR_HANDLE_TO_PTR(struct oxr_space *, equirect->space); + + enum xrt_layer_composition_flags flags = convert_layer_flags(equirect->layerFlags); + + struct xrt_pose *pose_ptr = (struct xrt_pose *)&equirect->pose; + + struct xrt_pose pose; + if (!handle_space(log, sess, spc, pose_ptr, inv_offset, timestamp, &pose)) { + return XR_SUCCESS; + } + + if (spc->is_reference && spc->type == XR_REFERENCE_SPACE_TYPE_VIEW) { + flags |= XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT; + } + + struct xrt_layer_data data; + U_ZERO(&data); + data.type = XRT_LAYER_EQUIRECT2; + data.name = XRT_INPUT_GENERIC_HEAD_POSE; + data.timestamp = timestamp; + data.flags = flags; + data.equirect2.visibility = convert_eye_visibility(equirect->eyeVisibility); + data.equirect2.pose = pose; + data.equirect2.radius = equirect->radius; + data.equirect2.central_horizontal_angle = equirect->centralHorizontalAngle; + data.equirect2.upper_vertical_angle = equirect->upperVerticalAngle; + data.equirect2.lower_vertical_angle = equirect->lowerVerticalAngle; + fill_in_sub_image(sc, &equirect->subImage, &data.equirect2.sub); + + CALL_CHK(xrt_comp_layer_equirect2(xc, head, sc->swapchain, &data)); + + return XR_SUCCESS; +} + +XrResult +oxr_session_frame_end(struct oxr_logger *log, struct oxr_session *sess, const XrFrameEndInfo *frameEndInfo) +{ + /* + * Session state and call order. + */ + + if (!is_session_running(sess)) { + return oxr_error(log, XR_ERROR_SESSION_NOT_RUNNING, "Session is not running"); + } + + if (!sess->frame_started) { + return oxr_error(log, XR_ERROR_CALL_ORDER_INVALID, "Frame not begun with xrBeginFrame"); + } + + if (frameEndInfo->displayTime <= 0) { + return oxr_error(log, XR_ERROR_TIME_INVALID, + "(frameEndInfo->displayTime == %" PRIi64 + ") zero or a negative value is not a valid XrTime", + frameEndInfo->displayTime); + } + + int64_t display_time_ns = + time_state_ts_to_monotonic_ns(sess->sys->inst->timekeeping, frameEndInfo->displayTime); + if (sess->frame_timing_spew) { + oxr_log(log, "End frame at %8.3fms with display time %8.3fms", ts_ms(sess), ns_to_ms(display_time_ns)); + } + + struct xrt_compositor *xc = sess->compositor; + + + /* + * Early out for headless sessions. + */ + if (xc == NULL) { + sess->frame_started = false; + + os_mutex_lock(&sess->active_wait_frames_lock); + sess->active_wait_frames--; + os_mutex_unlock(&sess->active_wait_frames_lock); + + do_synchronize_state_change(log, sess); + + return oxr_session_success_result(sess); + } + + + /* + * Blend mode. + * XR_ERROR_ENVIRONMENT_BLEND_MODE_UNSUPPORTED must always be reported, even with 0 layers. + */ + + enum xrt_blend_mode blend_mode = convert_blend_mode(frameEndInfo->environmentBlendMode); + struct xrt_device *xdev = GET_XDEV_BY_ROLE(sess->sys, head); + + if (!u_verify_blend_mode_valid(blend_mode)) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "(frameEndInfo->environmentBlendMode == 0x%08x) unknown environment blend mode", + frameEndInfo->environmentBlendMode); + } + + if (!u_verify_blend_mode_supported(xdev, blend_mode)) { + //! @todo Make integer print to string. + return oxr_error(log, XR_ERROR_ENVIRONMENT_BLEND_MODE_UNSUPPORTED, + "(frameEndInfo->environmentBlendMode == %u) is not supported", + frameEndInfo->environmentBlendMode); + } + + + /* + * Early out for discarded frame if layer count is 0. + */ + + if (frameEndInfo->layerCount == 0) { + + os_mutex_lock(&sess->active_wait_frames_lock); + sess->active_wait_frames--; + os_mutex_unlock(&sess->active_wait_frames_lock); + + CALL_CHK(xrt_comp_discard_frame(xc, sess->frame_id.begun)); + sess->frame_id.begun = -1; + sess->frame_started = false; + + do_synchronize_state_change(log, sess); + + return oxr_session_success_result(sess); + } + + + /* + * Layers. + */ + + if (frameEndInfo->layers == NULL) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, "(frameEndInfo->layers == NULL)"); + } + + for (uint32_t i = 0; i < frameEndInfo->layerCount; i++) { + const XrCompositionLayerBaseHeader *layer = frameEndInfo->layers[i]; + if (layer == NULL) { + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u] == NULL) layer can not be null", i); + } + + XrResult res; + + switch (layer->type) { + case XR_TYPE_COMPOSITION_LAYER_PROJECTION: + res = verify_projection_layer(xc, log, i, (XrCompositionLayerProjection *)layer, xdev, + frameEndInfo->displayTime); + break; + case XR_TYPE_COMPOSITION_LAYER_QUAD: + res = verify_quad_layer(xc, log, i, (XrCompositionLayerQuad *)layer, xdev, + frameEndInfo->displayTime); + break; + case XR_TYPE_COMPOSITION_LAYER_CUBE_KHR: + res = verify_cube_layer(xc, log, i, (XrCompositionLayerCubeKHR *)layer, xdev, + frameEndInfo->displayTime); + break; + case XR_TYPE_COMPOSITION_LAYER_CYLINDER_KHR: + res = verify_cylinder_layer(xc, log, i, (XrCompositionLayerCylinderKHR *)layer, xdev, + frameEndInfo->displayTime); + break; + case XR_TYPE_COMPOSITION_LAYER_EQUIRECT_KHR: + res = verify_equirect1_layer(xc, log, i, (XrCompositionLayerEquirectKHR *)layer, xdev, + frameEndInfo->displayTime); + break; + case XR_TYPE_COMPOSITION_LAYER_EQUIRECT2_KHR: + res = verify_equirect2_layer(xc, log, i, (XrCompositionLayerEquirect2KHR *)layer, xdev, + frameEndInfo->displayTime); + break; + default: + return oxr_error(log, XR_ERROR_LAYER_INVALID, + "(frameEndInfo->layers[%u]->type) layer type not supported (%u)", i, + layer->type); + } + + if (res != XR_SUCCESS) { + return res; + } + } + + + /* + * Done verifying. + */ + + // Do state change if needed. + do_synchronize_state_change(log, sess); + + struct xrt_pose inv_offset = {0}; + math_pose_invert(&xdev->tracking_origin->offset, &inv_offset); + + CALL_CHK(xrt_comp_layer_begin(xc, sess->frame_id.begun, display_time_ns, blend_mode)); + + for (uint32_t i = 0; i < frameEndInfo->layerCount; i++) { + const XrCompositionLayerBaseHeader *layer = frameEndInfo->layers[i]; + assert(layer != NULL); + + int64_t timestamp = + time_state_ts_to_monotonic_ns(sess->sys->inst->timekeeping, frameEndInfo->displayTime); + + switch (layer->type) { + case XR_TYPE_COMPOSITION_LAYER_PROJECTION: + submit_projection_layer(sess, xc, log, (XrCompositionLayerProjection *)layer, xdev, &inv_offset, + timestamp); + break; + case XR_TYPE_COMPOSITION_LAYER_QUAD: + submit_quad_layer(sess, xc, log, (XrCompositionLayerQuad *)layer, xdev, &inv_offset, timestamp); + break; + case XR_TYPE_COMPOSITION_LAYER_CUBE_KHR: + submit_cube_layer(sess, xc, log, (XrCompositionLayerCubeKHR *)layer, xdev, &inv_offset, + timestamp); + break; + case XR_TYPE_COMPOSITION_LAYER_CYLINDER_KHR: + submit_cylinder_layer(sess, xc, log, (XrCompositionLayerCylinderKHR *)layer, xdev, &inv_offset, + timestamp); + break; + case XR_TYPE_COMPOSITION_LAYER_EQUIRECT_KHR: + submit_equirect1_layer(sess, xc, log, (XrCompositionLayerEquirectKHR *)layer, xdev, &inv_offset, + timestamp); + break; + case XR_TYPE_COMPOSITION_LAYER_EQUIRECT2_KHR: + submit_equirect2_layer(sess, xc, log, (XrCompositionLayerEquirect2KHR *)layer, xdev, + &inv_offset, timestamp); + break; + default: assert(false && "invalid layer type"); + } + } + + CALL_CHK(xrt_comp_layer_commit(xc, sess->frame_id.begun, XRT_GRAPHICS_SYNC_HANDLE_INVALID)); + sess->frame_id.begun = -1; + + sess->frame_started = false; + + os_mutex_lock(&sess->active_wait_frames_lock); + sess->active_wait_frames--; + os_mutex_unlock(&sess->active_wait_frames_lock); + + return oxr_session_success_result(sess); +} diff --git a/src/xrt/state_trackers/oxr/oxr_session_egl.c b/src/xrt/state_trackers/oxr/oxr_session_gfx_egl.c similarity index 73% rename from src/xrt/state_trackers/oxr/oxr_session_egl.c rename to src/xrt/state_trackers/oxr/oxr_session_gfx_egl.c index b16246a41..75ff35ca4 100644 --- a/src/xrt/state_trackers/oxr/oxr_session_egl.c +++ b/src/xrt/state_trackers/oxr/oxr_session_gfx_egl.c @@ -1,4 +1,4 @@ -// Copyright 2018-2020, Collabora, Ltd. +// Copyright 2018-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -60,14 +60,21 @@ oxr_session_populate_egl(struct oxr_logger *log, } struct xrt_compositor_native *xcn = sess->xcn; - struct xrt_compositor_gl *xcgl = xrt_gfx_provider_create_gl_egl( // - xcn, // - next->display, // - next->config, // - next->context, // - next->getProcAddress); // + struct xrt_compositor_gl *xcgl = NULL; + xrt_result_t xret = xrt_gfx_provider_create_gl_egl( // + xcn, // + next->display, // + next->config, // + next->context, // + next->getProcAddress, // + &xcgl); // - if (xcgl == NULL) { + if (xret == XRT_ERROR_EGL_CONFIG_MISSING) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "XrGraphicsBindingEGLMNDX::config can not be null when EGL_KHR_no_config_context is " + "not supported by the display."); + } + if (xret != XRT_SUCCESS || xcgl == NULL) { return oxr_error(log, XR_ERROR_INITIALIZATION_FAILED, "Failed to create an egl client compositor"); } diff --git a/src/xrt/state_trackers/oxr/oxr_session_gl.c b/src/xrt/state_trackers/oxr/oxr_session_gfx_gl.c similarity index 100% rename from src/xrt/state_trackers/oxr/oxr_session_gl.c rename to src/xrt/state_trackers/oxr/oxr_session_gfx_gl.c diff --git a/src/xrt/state_trackers/oxr/oxr_session_gles_android.c b/src/xrt/state_trackers/oxr/oxr_session_gfx_gles_android.c similarity index 74% rename from src/xrt/state_trackers/oxr/oxr_session_gles_android.c rename to src/xrt/state_trackers/oxr/oxr_session_gfx_gles_android.c index 1b5b850ff..be55ab2d2 100644 --- a/src/xrt/state_trackers/oxr/oxr_session_gles_android.c +++ b/src/xrt/state_trackers/oxr/oxr_session_gfx_gles_android.c @@ -1,4 +1,4 @@ -// Copyright 2018-2020, Collabora, Ltd. +// Copyright 2018-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -68,14 +68,21 @@ oxr_session_populate_gles_android(struct oxr_logger *log, struct xrt_compositor_native *xcn = sess->xcn; - struct xrt_compositor_gl *xcgl = xrt_gfx_provider_create_gl_egl( // - xcn, // - next->display, // - next->config, // - next->context, // - get_proc_addr); // + struct xrt_compositor_gl *xcgl = NULL; + xrt_result_t xret = xrt_gfx_provider_create_gl_egl( // + xcn, // + next->display, // + next->config, // + next->context, // + get_proc_addr, // + &xcgl); // - if (xcgl == NULL) { + if (xret == XRT_ERROR_EGL_CONFIG_MISSING) { + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, + "XrGraphicsBindingEGLMNDX::config can not be null when EGL_KHR_no_config_context is " + "not supported by the display."); + } + if (xret != XR_SUCCESS || xcgl == NULL) { return oxr_error(log, XR_ERROR_INITIALIZATION_FAILED, "Failed to create an egl client compositor"); } diff --git a/src/xrt/state_trackers/oxr/oxr_session_vk.c b/src/xrt/state_trackers/oxr/oxr_session_gfx_vk.c similarity index 100% rename from src/xrt/state_trackers/oxr/oxr_session_vk.c rename to src/xrt/state_trackers/oxr/oxr_session_gfx_vk.c diff --git a/src/xrt/state_trackers/oxr/oxr_space.c b/src/xrt/state_trackers/oxr/oxr_space.c index 96600aba5..c1e4332b0 100644 --- a/src/xrt/state_trackers/oxr/oxr_space.c +++ b/src/xrt/state_trackers/oxr/oxr_space.c @@ -22,7 +22,7 @@ #include "oxr_handle.h" -const struct xrt_pose origin = {{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; +const struct xrt_pose origin = XRT_POSE_IDENTITY; static XrResult check_reference_space_type(struct oxr_logger *log, XrReferenceSpaceType type) diff --git a/src/xrt/state_trackers/oxr/oxr_swapchain.c b/src/xrt/state_trackers/oxr/oxr_swapchain.c index 43be3fefe..f6eac9369 100644 --- a/src/xrt/state_trackers/oxr/oxr_swapchain.c +++ b/src/xrt/state_trackers/oxr/oxr_swapchain.c @@ -1,4 +1,4 @@ -// Copyright 2018-2020, Collabora, Ltd. +// Copyright 2018-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -179,7 +179,8 @@ convert_usage_bits(XrSwapchainUsageFlags xr_usage) if ((xr_usage & XR_SWAPCHAIN_USAGE_MUTABLE_FORMAT_BIT) != 0) { usage |= XRT_SWAPCHAIN_USAGE_MUTABLE_FORMAT; } - if ((xr_usage & XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_MND) != 0) { + // aliased to XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_MND + if ((xr_usage & XR_SWAPCHAIN_USAGE_INPUT_ATTACHMENT_BIT_KHR) != 0) { usage |= XRT_SWAPCHAIN_USAGE_INPUT_ATTACHMENT; } @@ -205,7 +206,7 @@ oxr_create_swapchain(struct oxr_logger *log, info.array_size = createInfo->arraySize; info.mip_count = createInfo->mipCount; - struct xrt_swapchain *xsc = NULL; + struct xrt_swapchain *xsc = NULL; // Has to be NULL. xret = xrt_comp_create_swapchain(sess->compositor, &info, &xsc); if (xret == XRT_ERROR_SWAPCHAIN_FLAG_VALID_BUT_UNSUPPORTED) { return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, diff --git a/src/xrt/state_trackers/oxr/oxr_swapchain_gl.c b/src/xrt/state_trackers/oxr/oxr_swapchain_gl.c index 55746234a..7e2b858ef 100644 --- a/src/xrt/state_trackers/oxr/oxr_swapchain_gl.c +++ b/src/xrt/state_trackers/oxr/oxr_swapchain_gl.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -35,10 +35,8 @@ oxr_swapchain_gl_destroy(struct oxr_logger *log, struct oxr_swapchain *sc) sc->release_image(log, sc, NULL); } - if (sc->swapchain != NULL) { - sc->swapchain->destroy(sc->swapchain); - sc->swapchain = NULL; - } + // Drop our reference, does NULL checking. + xrt_swapchain_reference(&sc->swapchain, NULL); return XR_SUCCESS; } diff --git a/src/xrt/state_trackers/oxr/oxr_swapchain_vk.c b/src/xrt/state_trackers/oxr/oxr_swapchain_vk.c index 2a4199726..8b164ef3b 100644 --- a/src/xrt/state_trackers/oxr/oxr_swapchain_vk.c +++ b/src/xrt/state_trackers/oxr/oxr_swapchain_vk.c @@ -1,4 +1,4 @@ -// Copyright 2019-2020, Collabora, Ltd. +// Copyright 2019-2021, Collabora, Ltd. // SPDX-License-Identifier: BSL-1.0 /*! * @file @@ -32,10 +32,8 @@ oxr_swapchain_vk_destroy(struct oxr_logger *log, struct oxr_swapchain *sc) sc->release_image(log, sc, NULL); } - if (sc->swapchain != NULL) { - sc->swapchain->destroy(sc->swapchain); - sc->swapchain = NULL; - } + // Drop our reference, does NULL checking. + xrt_swapchain_reference(&sc->swapchain, NULL); return XR_SUCCESS; } diff --git a/src/xrt/state_trackers/oxr/oxr_system.c b/src/xrt/state_trackers/oxr/oxr_system.c index 8ed3a6bb8..ff0bd76e6 100644 --- a/src/xrt/state_trackers/oxr/oxr_system.c +++ b/src/xrt/state_trackers/oxr/oxr_system.c @@ -15,6 +15,7 @@ #include "xrt/xrt_device.h" #include "util/u_debug.h" +#include "util/u_verify.h" #include "oxr_objects.h" #include "oxr_logger.h" @@ -86,6 +87,8 @@ oxr_system_get_by_id(struct oxr_logger *log, struct oxr_instance *inst, XrSystem return XR_SUCCESS; } + + XrResult oxr_system_fill_in(struct oxr_logger *log, struct oxr_instance *inst, XrSystemId systemId, struct oxr_system *sys) { @@ -96,8 +99,10 @@ oxr_system_fill_in(struct oxr_logger *log, struct oxr_instance *inst, XrSystemId sys->form_factor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY; sys->view_config_type = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; +#ifdef XR_USE_GRAPHICS_API_VULKAN sys->vulkan_enable2_instance = VK_NULL_HANDLE; - sys->vulkan_enable2_physical_device = VK_NULL_HANDLE; + sys->suggested_vulkan_physical_device = VK_NULL_HANDLE; +#endif // Headless. if (sys->xsysc == NULL) { @@ -151,20 +156,16 @@ oxr_system_fill_in(struct oxr_logger *log, struct oxr_instance *inst, XrSystemId struct xrt_device *head = GET_XDEV_BY_ROLE(sys, head); - uint32_t i = 0; - if (head->hmd->blend_mode & XRT_BLEND_MODE_OPAQUE) { - sys->blend_modes[i++] = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; - } - if (head->hmd->blend_mode & XRT_BLEND_MODE_ADDITIVE) { - sys->blend_modes[i++] = XR_ENVIRONMENT_BLEND_MODE_ADDITIVE; - } - if (head->hmd->blend_mode & XRT_BLEND_MODE_ALPHA_BLEND) { - sys->blend_modes[i++] = XR_ENVIRONMENT_BLEND_MODE_ALPHA_BLEND; - } - sys->num_blend_modes = i; + assert(head->hmd->num_blend_modes <= XRT_MAX_DEVICE_BLEND_MODES); + assert(head->hmd->num_blend_modes != 0); - assert(i < ARRAY_SIZE(sys->blend_modes)); + for (size_t i = 0; i < head->hmd->num_blend_modes; i++) { + assert(u_verify_blend_mode_valid(head->hmd->blend_modes[i])); + sys->blend_modes[i] = (XrEnvironmentBlendMode)head->hmd->blend_modes[i]; + } + sys->num_blend_modes = (uint32_t)head->hmd->num_blend_modes; + assert(sys->num_blend_modes <= ARRAY_SIZE(sys->blend_modes)); return XR_SUCCESS; } @@ -193,21 +194,27 @@ oxr_system_get_properties(struct oxr_logger *log, struct oxr_system *sys, XrSyst snprintf(properties->systemName, XR_MAX_SYSTEM_NAME_SIZE, "Monado: %.*s", 247, xdev->str); // Get from compositor. - struct xrt_system_compositor_info *info = &sys->xsysc->info; + struct xrt_system_compositor_info *info = sys->xsysc ? &sys->xsysc->info : NULL; - properties->graphicsProperties.maxLayerCount = info->max_layers; + if (info) { + properties->graphicsProperties.maxLayerCount = info->max_layers; + } else { + // probably using the headless extension, but the extension does not modify the 16 layer minimum. + properties->graphicsProperties.maxLayerCount = 16; + } properties->graphicsProperties.maxSwapchainImageWidth = 1024 * 16; properties->graphicsProperties.maxSwapchainImageHeight = 1024 * 16; properties->trackingProperties.orientationTracking = xdev->orientation_tracking_supported; properties->trackingProperties.positionTracking = xdev->position_tracking_supported; - XrSystemHandTrackingPropertiesEXT *hand_tracking_props = OXR_GET_OUTPUT_FROM_CHAIN( - properties, XR_TYPE_SYSTEM_HAND_TRACKING_PROPERTIES_EXT, XrSystemHandTrackingPropertiesEXT); + XrSystemHandTrackingPropertiesEXT *hand_tracking_props = NULL; + // We should only be looking for extension structs if the extension has been enabled. + if (sys->inst->extensions.EXT_hand_tracking) { + hand_tracking_props = OXR_GET_OUTPUT_FROM_CHAIN(properties, XR_TYPE_SYSTEM_HAND_TRACKING_PROPERTIES_EXT, + XrSystemHandTrackingPropertiesEXT); + } if (hand_tracking_props) { - if (!sys->inst->extensions.EXT_hand_tracking) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "XR_EXT_hand_tracking is not enabled."); - } hand_tracking_props->supportsHandTracking = oxr_system_get_hand_tracking_support(log, sys->inst); } diff --git a/src/xrt/state_trackers/oxr/oxr_verify.c b/src/xrt/state_trackers/oxr/oxr_verify.c index eade7f524..c70abfbfb 100644 --- a/src/xrt/state_trackers/oxr/oxr_verify.c +++ b/src/xrt/state_trackers/oxr/oxr_verify.c @@ -564,9 +564,7 @@ oxr_verify_XrGraphicsBindingEGLMNDX(struct oxr_logger *log, const XrGraphicsBind return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "XrGraphicsBindingEGLMNDX::display cannot be NULL"); } - if (next->config == NULL) { - return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "XrGraphicsBindingEGLMNDX::config cannot be NULL"); - } + // The next->config field can be NULL if EGL_KHR_no_config_context is supported by the display. if (next->context == NULL) { return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "XrGraphicsBindingEGLMNDX::context cannot be NULL"); diff --git a/src/xrt/state_trackers/oxr/oxr_vulkan.c b/src/xrt/state_trackers/oxr/oxr_vulkan.c index 6cc93d0f7..4d6498543 100644 --- a/src/xrt/state_trackers/oxr/oxr_vulkan.c +++ b/src/xrt/state_trackers/oxr/oxr_vulkan.c @@ -369,7 +369,10 @@ oxr_vk_get_physical_device(struct oxr_logger *log, // vulkan_enable2 needs the physical device in xrCreateVulkanDeviceKHR if (inst->extensions.KHR_vulkan_enable2) { sys->vulkan_enable2_instance = vkInstance; - sys->vulkan_enable2_physical_device = *vkPhysicalDevice; + } + sys->suggested_vulkan_physical_device = *vkPhysicalDevice; + if (ll <= U_LOGGING_DEBUG) { + oxr_log(log, "Suggesting vulkan physical device %p", (void *)*vkPhysicalDevice); } free(phys); diff --git a/src/xrt/state_trackers/oxr/oxr_xdev.c b/src/xrt/state_trackers/oxr/oxr_xdev.c index 2df44cc1a..6cc92eb8c 100644 --- a/src/xrt/state_trackers/oxr/oxr_xdev.c +++ b/src/xrt/state_trackers/oxr/oxr_xdev.c @@ -90,6 +90,7 @@ oxr_xdev_get_space_graph(struct oxr_logger *log, uint64_t at_timestamp_ns = time_state_ts_to_monotonic_ns(inst->timekeeping, at_time); struct xrt_space_relation *rel = m_space_graph_reserve(xsg); + xrt_device_get_tracked_pose(xdev, name, at_timestamp_ns, rel); // Add in the offset from the tracking system. @@ -104,12 +105,15 @@ oxr_xdev_get_hand_tracking_at(struct oxr_logger *log, XrTime at_time, struct xrt_hand_joint_set *out_value) { + //! @todo Moses doesn't know what he's doing here! //! Convert at_time to monotonic and give to device. uint64_t at_timestamp_ns = time_state_ts_to_monotonic_ns(inst->timekeeping, at_time); struct xrt_hand_joint_set value; - xrt_device_get_hand_tracking(xdev, name, at_timestamp_ns, &value); + uint64_t ignored; + + xrt_device_get_hand_tracking(xdev, name, at_timestamp_ns, &value, &ignored); *out_value = value; } diff --git a/src/xrt/state_trackers/prober/CMakeLists.txt b/src/xrt/state_trackers/prober/CMakeLists.txt index 33ce97a97..4f57bcdd5 100644 --- a/src/xrt/state_trackers/prober/CMakeLists.txt +++ b/src/xrt/state_trackers/prober/CMakeLists.txt @@ -6,7 +6,6 @@ set(PROBER_INCLUDES) set(PROBER_SOURCE_FILES p_documentation.h p_dump.c - p_json.c p_prober.c p_prober.h p_tracking.c @@ -40,6 +39,7 @@ target_link_libraries(st_prober PUBLIC xrt-interfaces ) target_link_libraries(st_prober PRIVATE + drv_multi aux_util aux_os aux_tracking @@ -89,4 +89,4 @@ if(XRT_BUILD_DRIVER_REMOTE) target_link_libraries(st_prober PRIVATE drv_remote ) -endif() \ No newline at end of file +endif() diff --git a/src/xrt/state_trackers/prober/meson.build b/src/xrt/state_trackers/prober/meson.build index 18ffff078..d9c5bf990 100644 --- a/src/xrt/state_trackers/prober/meson.build +++ b/src/xrt/state_trackers/prober/meson.build @@ -4,7 +4,6 @@ prober_sources = [ 'p_documentation.h', 'p_dump.c', - 'p_json.c', 'p_prober.c', 'p_prober.h', 'p_tracking.c', diff --git a/src/xrt/state_trackers/prober/p_dump.c b/src/xrt/state_trackers/prober/p_dump.c index feca050c0..d710a9bed 100644 --- a/src/xrt/state_trackers/prober/p_dump.c +++ b/src/xrt/state_trackers/prober/p_dump.c @@ -71,34 +71,34 @@ p_dump_device(struct prober *p, struct prober_device *pdev, int id) return; } - U_LOG_I("\t% 3i: 0x%04x:0x%04x", id, pdev->base.vendor_id, pdev->base.product_id); - U_LOG_I("\t\tptr: %p", (void *)pdev); - U_LOG_I("\t\tusb_dev_class: %02x", pdev->base.usb_dev_class); + U_LOG_RAW("\t% 3i: 0x%04x:0x%04x", id, pdev->base.vendor_id, pdev->base.product_id); + U_LOG_RAW("\t\tptr: %p", (void *)pdev); + U_LOG_RAW("\t\tusb_dev_class: %02x", pdev->base.usb_dev_class); if (pdev->usb.serial != NULL || pdev->usb.product != NULL || pdev->usb.manufacturer != NULL) { - U_LOG_I("\t\tusb.product: %s", pdev->usb.product); - U_LOG_I("\t\tusb.manufacturer: %s", pdev->usb.manufacturer); - U_LOG_I("\t\tusb.serial: %s", pdev->usb.serial); + U_LOG_RAW("\t\tusb.product: %s", pdev->usb.product); + U_LOG_RAW("\t\tusb.manufacturer: %s", pdev->usb.manufacturer); + U_LOG_RAW("\t\tusb.serial: %s", pdev->usb.serial); } if (pdev->usb.bus != 0 || pdev->usb.addr != 0) { - U_LOG_I("\t\tusb.bus: %i", pdev->usb.bus); - U_LOG_I("\t\tusb.addr: %i", pdev->usb.addr); + U_LOG_RAW("\t\tusb.bus: %i", pdev->usb.bus); + U_LOG_RAW("\t\tusb.addr: %i", pdev->usb.addr); } if (pdev->bluetooth.id != 0) { - U_LOG_I("\t\tbluetooth.id: %012" PRIx64 "", pdev->bluetooth.id); + U_LOG_RAW("\t\tbluetooth.id: %012" PRIx64 "", pdev->bluetooth.id); } int num = pdev->usb.num_ports; if (print_ports(tmp, ARRAY_SIZE(tmp), pdev->usb.ports, num)) { - U_LOG_I("\t\tport%s %s", num > 1 ? "s:" : ": ", tmp); + U_LOG_RAW("\t\tport%s %s", num > 1 ? "s:" : ": ", tmp); } #ifdef XRT_HAVE_LIBUSB if (pdev->usb.dev != NULL) { - U_LOG_I("\t\tlibusb: %p", (void *)pdev->usb.dev); + U_LOG_RAW("\t\tlibusb: %p", (void *)pdev->usb.dev); } #endif @@ -107,21 +107,21 @@ p_dump_device(struct prober *p, struct prober_device *pdev, int id) if (uvc_dev != NULL) { struct uvc_device_descriptor *desc; - U_LOG_I("\t\tlibuvc: %p", (void *)uvc_dev); + U_LOG_RAW("\t\tlibuvc: %p", (void *)uvc_dev); uvc_get_device_descriptor(uvc_dev, &desc); if (desc->product != NULL) { - U_LOG_I("\t\tproduct: '%s'", desc->product); + U_LOG_RAW("\t\tproduct: '%s'", desc->product); } if (desc->manufacturer != NULL) { - U_LOG_I("\t\tmanufacturer: '%s'", desc->manufacturer); + U_LOG_RAW("\t\tmanufacturer: '%s'", desc->manufacturer); } if (desc->serialNumber != NULL) { - U_LOG_I("\t\tserial: '%s'", desc->serialNumber); + U_LOG_RAW("\t\tserial: '%s'", desc->serialNumber); } uvc_free_device_descriptor(desc); @@ -133,9 +133,9 @@ p_dump_device(struct prober *p, struct prober_device *pdev, int id) for (size_t j = 0; j < pdev->num_v4ls; j++) { struct prober_v4l *v4l = &pdev->v4ls[j]; - U_LOG_I("\t\tv4l.iface: %i", (int)v4l->usb_iface); - U_LOG_I("\t\tv4l.index: %i", (int)v4l->v4l_index); - U_LOG_I("\t\tv4l.path: '%s'", v4l->path); + U_LOG_RAW("\t\tv4l.iface: %i", (int)v4l->usb_iface); + U_LOG_RAW("\t\tv4l.index: %i", (int)v4l->v4l_index); + U_LOG_RAW("\t\tv4l.path: '%s'", v4l->path); } #endif @@ -143,8 +143,8 @@ p_dump_device(struct prober *p, struct prober_device *pdev, int id) for (size_t j = 0; j < pdev->num_hidraws; j++) { struct prober_hidraw *hidraw = &pdev->hidraws[j]; - U_LOG_I("\t\thidraw.iface: %i", (int)hidraw->interface); - U_LOG_I("\t\thidraw.path: '%s'", hidraw->path); + U_LOG_RAW("\t\thidraw.iface: %i", (int)hidraw->interface); + U_LOG_RAW("\t\thidraw.path: '%s'", hidraw->path); } #endif } diff --git a/src/xrt/state_trackers/prober/p_json.c b/src/xrt/state_trackers/prober/p_json.c deleted file mode 100644 index d220c3552..000000000 --- a/src/xrt/state_trackers/prober/p_json.c +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright 2019, Collabora, Ltd. -// SPDX-License-Identifier: BSL-1.0 -/*! - * @file - * @brief Code to manage the settings file. - * @author Jakob Bornecrantz - * @ingroup st_prober - */ - -#include "util/u_file.h" -#include "util/u_json.h" -#include "util/u_debug.h" - -#include "p_prober.h" - -#include -#include -#include -#include - -DEBUG_GET_ONCE_OPTION(active_config, "P_OVERRIDE_ACTIVE_CONFIG", NULL) - - -char * -read_content(FILE *file) -{ - // Go to the end of the file. - fseek(file, 0L, SEEK_END); - size_t file_size = ftell(file); - - // Return back to the start of the file. - fseek(file, 0L, SEEK_SET); - - char *buffer = (char *)calloc(file_size + 1, sizeof(char)); - if (buffer == NULL) { - return NULL; - } - - // Do the actual reading. - size_t ret = fread(buffer, sizeof(char), file_size, file); - if (ret != file_size) { - free(buffer); - return NULL; - } - - return buffer; -} - -void -p_json_open_or_create_main_file(struct prober *p) -{ -#ifdef XRT_OS_LINUX - char tmp[1024]; - ssize_t ret = u_file_get_path_in_config_dir("config_v0.json", tmp, sizeof(tmp)); - if (ret <= 0) { - U_LOG_E( - "Could not load or create config file no $HOME " - "or $XDG_CONFIG_HOME env variables defined"); - return; - } - - FILE *file = u_file_open_file_in_config_dir("config_v0.json", "r"); - if (file == NULL) { - return; - } - - p->json.file_loaded = true; - - char *str = read_content(file); - fclose(file); - if (str == NULL) { - U_LOG_E("Could not read the contents of '%s'!", tmp); - return; - } - - // No config created, ignore. - if (strlen(str) == 0) { - free(str); - return; - } - - p->json.root = cJSON_Parse(str); - if (p->json.root == NULL) { - U_LOG_E("Failed to parse JSON in '%s':\n%s\n#######", tmp, str); - U_LOG_E("'%s'", cJSON_GetErrorPtr()); - } - - free(str); -#else - //! @todo implement the underlying u_file_get_path_in_config_dir - return; -#endif -} - -static cJSON * -get_obj(cJSON *json, const char *name) -{ - cJSON *item = cJSON_GetObjectItemCaseSensitive(json, name); - if (item == NULL) { - U_LOG_E("Failed to find node '%s'!", name); - } - return item; -} - -XRT_MAYBE_UNUSED static bool -get_obj_bool(cJSON *json, const char *name, bool *out_bool) -{ - cJSON *item = get_obj(json, name); - if (item == NULL) { - return false; - } - - if (!u_json_get_bool(item, out_bool)) { - U_LOG_E("Failed to parse '%s'!", name); - return false; - } - - return true; -} - -static bool -get_obj_int(cJSON *json, const char *name, int *out_int) -{ - cJSON *item = get_obj(json, name); - if (item == NULL) { - return false; - } - - if (!u_json_get_int(item, out_int)) { - U_LOG_E("Failed to parse '%s'!", name); - return false; - } - - return true; -} - -static bool -get_obj_str(cJSON *json, const char *name, char *array, size_t array_size) -{ - cJSON *item = get_obj(json, name); - if (item == NULL) { - return false; - } - - if (!u_json_get_string_into_array(item, array, array_size)) { - U_LOG_E("Failed to parse '%s'!", name); - return false; - } - - return true; -} - -static bool -is_json_ok(struct prober *p) -{ - if (p->json.root == NULL) { - if (p->json.file_loaded) { - U_LOG_E("JSON not parsed!"); - } else { - U_LOG_W("No config file!"); - } - return false; - } - - return true; -} - -static bool -parse_active(const char *str, const char *from, enum p_active_config *out_active) -{ - if (strcmp(str, "none") == 0) { - *out_active = P_ACTIVE_CONFIG_NONE; - } else if (strcmp(str, "tracking") == 0) { - *out_active = P_ACTIVE_CONFIG_TRACKING; - } else if (strcmp(str, "remote") == 0) { - *out_active = P_ACTIVE_CONFIG_REMOTE; - } else { - U_LOG_E("Unknown active config '%s' from %s.", str, from); - *out_active = P_ACTIVE_CONFIG_NONE; - return false; - } - - return true; -} - -void -p_json_get_active(struct prober *p, enum p_active_config *out_active) -{ - const char *str = debug_get_option_active_config(); - if (str != NULL && parse_active(str, "environment", out_active)) { - return; - } - - char tmp[256]; - if (!is_json_ok(p) || !get_obj_str(p->json.root, "active", tmp, sizeof(tmp))) { - *out_active = P_ACTIVE_CONFIG_NONE; - return; - } - - parse_active(tmp, "json", out_active); -} - -bool -p_json_get_remote_port(struct prober *p, int *out_port) -{ - cJSON *t = cJSON_GetObjectItemCaseSensitive(p->json.root, "remote"); - if (t == NULL) { - U_LOG_E("No remote node"); - return false; - } - - int ver = -1; - if (!get_obj_int(t, "version", &ver)) { - U_LOG_E("Missing version tag!"); - return false; - } - if (ver >= 1) { - U_LOG_E("Unknown version tag '%i'!", ver); - return false; - } - - int port = 0; - if (!get_obj_int(t, "port", &port)) { - return false; - } - - *out_port = port; - - return true; -} - -bool -p_json_get_tracking_settings(struct prober *p, struct xrt_settings_tracking *s) -{ - if (p->json.root == NULL) { - if (p->json.file_loaded) { - U_LOG_E("JSON not parsed!"); - } else { - U_LOG_W("No config file!"); - } - return false; - } - - cJSON *t = cJSON_GetObjectItemCaseSensitive(p->json.root, "tracking"); - if (t == NULL) { - U_LOG_E("No tracking node"); - return false; - } - - char tmp[16]; - - int ver = -1; - bool bad = false; - - bad |= !get_obj_int(t, "version", &ver); - if (bad || ver >= 1) { - U_LOG_E("Missing or unknown version tag '%i'", ver); - return false; - } - - bad |= !get_obj_str(t, "camera_name", s->camera_name, sizeof(s->camera_name)); - bad |= !get_obj_int(t, "camera_mode", &s->camera_mode); - bad |= !get_obj_str(t, "camera_type", tmp, sizeof(tmp)); - bad |= !get_obj_str(t, "calibration_path", s->calibration_path, sizeof(s->calibration_path)); - if (bad) { - return false; - } - - if (strcmp(tmp, "regular_mono") == 0) { - s->camera_type = XRT_SETTINGS_CAMERA_TYPE_REGULAR_MONO; - } else if (strcmp(tmp, "regular_sbs") == 0) { - s->camera_type = XRT_SETTINGS_CAMERA_TYPE_REGULAR_SBS; - } else if (strcmp(tmp, "ps4") == 0) { - s->camera_type = XRT_SETTINGS_CAMERA_TYPE_PS4; - } else if (strcmp(tmp, "leap_motion") == 0) { - s->camera_type = XRT_SETTINGS_CAMERA_TYPE_LEAP_MOTION; - } else { - U_LOG_W("Unknown camera type '%s'", tmp); - return false; - } - - return true; -} diff --git a/src/xrt/state_trackers/prober/p_prober.c b/src/xrt/state_trackers/prober/p_prober.c index 1fc0f2455..1639df0c0 100644 --- a/src/xrt/state_trackers/prober/p_prober.c +++ b/src/xrt/state_trackers/prober/p_prober.c @@ -8,11 +8,14 @@ */ #include "xrt/xrt_config_drivers.h" +#include "xrt/xrt_settings.h" #include "util/u_var.h" #include "util/u_misc.h" -#include "util/u_json.h" +#include "util/u_config_json.h" #include "util/u_debug.h" +#include "util/u_trace_marker.h" + #include "os/os_hid.h" #include "p_prober.h" @@ -20,10 +23,14 @@ #include "v4l2/v4l2_interface.h" #endif -#ifdef XRT_HAVE_VF +#ifdef XRT_BUILD_DRIVER_VF #include "vf/vf_interface.h" #endif +#ifdef XRT_BUILD_DRIVER_EUROC +#include "euroc/euroc_interface.h" +#endif + #ifdef XRT_BUILD_DRIVER_REMOTE #include "remote/r_interface.h" #endif @@ -32,6 +39,21 @@ #include #include +#include "multi_wrapper/multi.h" + + +/* + * + * Env variable options. + * + */ + +DEBUG_GET_ONCE_LOG_OPTION(prober_log, "PROBER_LOG", U_LOGGING_INFO) +DEBUG_GET_ONCE_BOOL_OPTION(qwerty_enable, "QWERTY_ENABLE", false) +DEBUG_GET_ONCE_BOOL_OPTION(qwerty_combine, "QWERTY_COMBINE", false) +DEBUG_GET_ONCE_OPTION(vf_path, "VF_PATH", NULL) +DEBUG_GET_ONCE_OPTION(euroc_path, "EUROC_PATH", NULL) + /* * @@ -39,8 +61,6 @@ * */ -DEBUG_GET_ONCE_LOG_OPTION(prober_log, "PROBER_LOG", U_LOGGING_WARN) - static void add_device(struct prober *p, struct prober_device **out_dev); @@ -54,40 +74,47 @@ static void teardown(struct prober *p); static int -probe(struct xrt_prober *xp); +p_probe(struct xrt_prober *xp); static int -dump(struct xrt_prober *xp); +p_dump(struct xrt_prober *xp); static int -select_device(struct xrt_prober *xp, struct xrt_device **xdevs, size_t num_xdevs); +p_select_device(struct xrt_prober *xp, struct xrt_device **xdevs, size_t num_xdevs); static int -open_hid_interface(struct xrt_prober *xp, - struct xrt_prober_device *xpdev, - int interface, - struct os_hid_device **out_hid_dev); +p_open_hid_interface(struct xrt_prober *xp, + struct xrt_prober_device *xpdev, + int interface, + struct os_hid_device **out_hid_dev); static int -open_video_device(struct xrt_prober *xp, - struct xrt_prober_device *xpdev, - struct xrt_frame_context *xfctx, - struct xrt_fs **out_xfs); +p_open_video_device(struct xrt_prober *xp, + struct xrt_prober_device *xpdev, + struct xrt_frame_context *xfctx, + struct xrt_fs **out_xfs); static int -list_video_devices(struct xrt_prober *xp, xrt_prober_list_video_cb cb, void *ptr); +p_list_video_devices(struct xrt_prober *xp, xrt_prober_list_video_cb cb, void *ptr); + static int -get_string_descriptor(struct xrt_prober *xp, - struct xrt_prober_device *xpdev, - enum xrt_prober_string which_string, - unsigned char *buffer, - int length); +p_get_entries(struct xrt_prober *xp, + size_t *out_num_entries, + struct xrt_prober_entry ***out_entries, + struct xrt_auto_prober ***out_auto_probers); + +static int +p_get_string_descriptor(struct xrt_prober *xp, + struct xrt_prober_device *xpdev, + enum xrt_prober_string which_string, + unsigned char *buffer, + size_t length); static bool -can_open(struct xrt_prober *xp, struct xrt_prober_device *xpdev); +p_can_open(struct xrt_prober *xp, struct xrt_prober_device *xpdev); static void -destroy(struct xrt_prober **xp); +p_destroy(struct xrt_prober **xp); /* @@ -145,13 +172,13 @@ xrt_prober_match_string(struct xrt_prober *xp, { unsigned char s[256] = {0}; int len = xrt_prober_get_string_descriptor(xp, dev, type, s, sizeof(s)); - if (len == 0) + if (len <= 0) { return false; + } return 0 == strncmp(to_match, (const char *)s, sizeof(s)); } - int p_dev_get_usb_dev(struct prober *p, uint16_t bus, @@ -298,27 +325,126 @@ collect_entries(struct prober *p) return 0; } + +#define num_driver_conflicts 1 +char *driver_conflicts[num_driver_conflicts][2] = {{"survive", "vive"}}; + +static void +disable_drivers_from_conflicts(struct prober *p) +{ + if (debug_get_bool_option_qwerty_enable() && !debug_get_bool_option_qwerty_combine()) { + for (size_t entry = 0; entry < p->num_entries; entry++) { + if (strcmp(p->entries[entry]->driver_name, "Qwerty") != 0) { + P_INFO(p, "Disabling %s because we have %s", p->entries[entry]->driver_name, "Qwerty"); + size_t index = p->num_disabled_drivers++; + U_ARRAY_REALLOC_OR_FREE(p->disabled_drivers, char *, p->num_disabled_drivers); + p->disabled_drivers[index] = (char *)p->entries[entry]->driver_name; + } + } + return; + } + + for (size_t i = 0; i < num_driver_conflicts; i++) { + bool have_first = false; + bool have_second = false; + + char *first = driver_conflicts[i][0]; + char *second = driver_conflicts[i][1]; + + // disable second driver if we have first driver + for (size_t entry = 0; entry < p->num_entries; entry++) { + if (strcmp(p->entries[entry]->driver_name, first) == 0) { + have_first = true; + } + if (strcmp(p->entries[entry]->driver_name, second) == 0) { + have_second = true; + } + } + + for (size_t ap = 0; ap < MAX_AUTO_PROBERS; ap++) { + if (p->auto_probers[ap] == NULL) { + continue; + } + if (strcmp(p->auto_probers[ap]->name, first) == 0) { + have_first = true; + } + if (strcmp(p->auto_probers[ap]->name, second) == 0) { + have_second = true; + } + } + + if (have_first && have_second) { + + // except don't disable second driver, if first driver is already disabled' + bool first_already_disabled = false; + ; + for (size_t disabled = 0; disabled < p->num_disabled_drivers; disabled++) { + if (strcmp(p->disabled_drivers[disabled], first) == 0) { + first_already_disabled = true; + break; + } + } + if (first_already_disabled) { + P_INFO(p, "Not disabling %s because %s is disabled", second, first); + continue; + } + + P_INFO(p, "Disabling %s because we have %s", second, first); + size_t index = p->num_disabled_drivers++; + U_ARRAY_REALLOC_OR_FREE(p->disabled_drivers, char *, p->num_disabled_drivers); + p->disabled_drivers[index] = second; + } + } +} + +static void +parse_disabled_drivers(struct prober *p) +{ + cJSON *disabled_drivers = cJSON_GetObjectItemCaseSensitive(p->json.root, "disabled"); + if (!disabled_drivers) { + return; + } + + cJSON *disabled_driver = NULL; + cJSON_ArrayForEach(disabled_driver, disabled_drivers) + { + if (!cJSON_IsString(disabled_driver)) { + continue; + } + + size_t index = p->num_disabled_drivers++; + U_ARRAY_REALLOC_OR_FREE(p->disabled_drivers, char *, p->num_disabled_drivers); + p->disabled_drivers[index] = disabled_driver->valuestring; + } +} + static int initialize(struct prober *p, struct xrt_prober_entry_lists *lists) { - p->base.probe = probe; - p->base.dump = dump; - p->base.select = select_device; - p->base.open_hid_interface = open_hid_interface; - p->base.open_video_device = open_video_device; - p->base.list_video_devices = list_video_devices; - p->base.get_string_descriptor = get_string_descriptor; - p->base.can_open = can_open; - p->base.destroy = destroy; + XRT_TRACE_MARKER(); + + p->base.probe = p_probe; + p->base.dump = p_dump; + p->base.select = p_select_device; + p->base.open_hid_interface = p_open_hid_interface; + p->base.open_video_device = p_open_video_device; + p->base.list_video_devices = p_list_video_devices; + p->base.get_entries = p_get_entries; + p->base.get_string_descriptor = p_get_string_descriptor; + p->base.can_open = p_can_open; + p->base.destroy = p_destroy; p->lists = lists; p->ll = debug_get_log_option_prober_log(); + p->json.file_loaded = false; + p->json.root = NULL; + u_var_add_root((void *)p, "Prober", true); - u_var_add_ro_u32(p, &p->ll, "Log Level"); + u_var_add_ro_u32(p, (uint32_t *)&p->ll, "Log Level"); int ret; - p_json_open_or_create_main_file(p); + u_config_json_open_or_create_main_file(&p->json); ret = collect_entries(p); if (ret != 0) { @@ -352,12 +478,19 @@ initialize(struct prober *p, struct xrt_prober_entry_lists *lists) p->auto_probers[i] = lists->auto_probers[i](); } + + p->num_disabled_drivers = 0; + parse_disabled_drivers(p); + disable_drivers_from_conflicts(p); + return 0; } static void teardown_devices(struct prober *p) { + XRT_TRACE_MARKER(); + // Need to free all devices. for (size_t i = 0; i < p->num_devices; i++) { struct prober_device *pdev = &p->devices[i]; @@ -433,6 +566,8 @@ teardown_devices(struct prober *p) static void teardown(struct prober *p) { + XRT_TRACE_MARKER(); + // First remove the variable tracking. u_var_remove_root((void *)p); @@ -462,68 +597,9 @@ teardown(struct prober *p) p_libusb_teardown(p); #endif - if (p->json.root != NULL) { - cJSON_Delete(p->json.root); - p->json.root = NULL; - } -} + u_config_json_close(&p->json); - -/* - * - * Member functions. - * - */ - -static int -probe(struct xrt_prober *xp) -{ - struct prober *p = (struct prober *)xp; - XRT_MAYBE_UNUSED int ret = 0; - - // Free old list first. - teardown_devices(p); - -#ifdef XRT_HAVE_LIBUDEV - ret = p_udev_probe(p); - if (ret != 0) { - P_ERROR(p, "Failed to enumerate udev devices\n"); - return -1; - } -#endif - -#ifdef XRT_HAVE_LIBUSB - ret = p_libusb_probe(p); - if (ret != 0) { - P_ERROR(p, "Failed to enumerate libusb devices\n"); - return -1; - } -#endif - -#ifdef XRT_HAVE_LIBUVC - ret = p_libuvc_probe(p); - if (ret != 0) { - P_ERROR(p, "Failed to enumerate libuvc devices\n"); - return -1; - } -#endif - - return 0; -} - -static int -dump(struct xrt_prober *xp) -{ - struct prober *p = (struct prober *)xp; - XRT_MAYBE_UNUSED ssize_t k = 0; - XRT_MAYBE_UNUSED size_t j = 0; - - for (size_t i = 0; i < p->num_devices; i++) { - struct prober_device *pdev = &p->devices[i]; - p_dump_device(p, pdev, (int)i); - } - - return 0; + free(p->disabled_drivers); } static void @@ -576,6 +652,19 @@ add_from_devices(struct prober *p, struct xrt_device **xdevs, size_t num_xdevs, continue; } + bool skip = false; + for (size_t disabled = 0; disabled < p->num_disabled_drivers; disabled++) { + if (strcmp(entry->driver_name, p->disabled_drivers[disabled]) == 0) { + P_INFO(p, "Skipping disabled driver %s", entry->driver_name); + skip = true; + break; + ; + } + } + if (skip) { + continue; + } + struct xrt_device *new_xdevs[XRT_MAX_DEVICES_PER_PROBE] = {NULL}; int num_found = entry->found(&p->base, dev_list, p->num_devices, i, NULL, &(new_xdevs[0])); @@ -604,6 +693,19 @@ static void add_from_auto_probers(struct prober *p, struct xrt_device **xdevs, size_t num_xdevs, bool *have_hmd) { for (int i = 0; i < MAX_AUTO_PROBERS && p->auto_probers[i]; i++) { + + bool skip = false; + for (size_t disabled = 0; disabled < p->num_disabled_drivers; disabled++) { + if (strcmp(p->auto_probers[i]->name, p->disabled_drivers[disabled]) == 0) { + P_INFO(p, "Skipping disabled driver %s", p->auto_probers[i]->name); + skip = true; + break; + } + } + if (skip) { + continue; + } + /* * If we have found a HMD, tell the auto probers not to open * any more HMDs. This is mostly to stop OpenHMD and Monado @@ -611,13 +713,24 @@ add_from_auto_probers(struct prober *p, struct xrt_device **xdevs, size_t num_xd */ bool no_hmds = *have_hmd; - struct xrt_device *xdev = - p->auto_probers[i]->lelo_dallas_autoprobe(p->auto_probers[i], NULL, no_hmds, &p->base); - if (xdev == NULL) { + struct xrt_device *new_xdevs[XRT_MAX_DEVICES_PER_PROBE] = {NULL}; + int num_found = + p->auto_probers[i]->lelo_dallas_autoprobe(p->auto_probers[i], NULL, no_hmds, &p->base, new_xdevs); + + if (num_found <= 0) { continue; } - handle_found_device(p, xdevs, num_xdevs, have_hmd, xdev); + for (int created_idx = 0; created_idx < num_found; ++created_idx) { + if (new_xdevs[created_idx] == NULL) { + P_DEBUG(p, + "Leaving device creation loop early: %s autoprobe function reported %i " + "created, but only %i non-null", + p->auto_probers[i]->name, num_found, created_idx); + continue; + } + handle_found_device(p, xdevs, num_xdevs, have_hmd, new_xdevs[created_idx]); + } } } @@ -630,7 +743,7 @@ add_from_remote(struct prober *p, struct xrt_device **xdevs, size_t num_xdevs, b #ifdef XRT_BUILD_DRIVER_REMOTE int port = 4242; - if (!p_json_get_remote_port(p, &port)) { + if (!u_config_json_get_remote_port(&p->json, &port)) { port = 4242; } @@ -639,22 +752,133 @@ add_from_remote(struct prober *p, struct xrt_device **xdevs, size_t num_xdevs, b #endif } -static int -select_device(struct xrt_prober *xp, struct xrt_device **xdevs, size_t num_xdevs) +static void +apply_tracking_override(struct prober *p, struct xrt_device **xdevs, size_t num_xdevs, struct xrt_tracking_override *o) { + struct xrt_device *target_xdev = NULL; + size_t target_idx = 0; + struct xrt_device *tracker_xdev = NULL; + + for (size_t i = 0; i < num_xdevs; i++) { + struct xrt_device *xdev = xdevs[i]; + if (xdev == NULL) { + continue; + } + + if (strncmp(xdev->serial, o->target_device_serial, XRT_DEVICE_NAME_LEN) == 0) { + target_xdev = xdev; + target_idx = i; + } + if (strncmp(xdev->serial, o->tracker_device_serial, XRT_DEVICE_NAME_LEN) == 0) { + tracker_xdev = xdev; + } + } + + if (target_xdev == NULL) { + P_WARN(p, "Tracking override target xdev %s not found", o->target_device_serial); + } + + if (tracker_xdev == NULL) { + P_WARN(p, "Tracking override tracker xdev %s not found", o->tracker_device_serial); + } + + + if (target_xdev != NULL && tracker_xdev != NULL) { + struct xrt_device *multi = multi_create_tracking_override(o->override_type, target_xdev, tracker_xdev, + o->input_name, &o->offset); + + if (multi) { + P_INFO(p, "Applying Tracking override %s <- %s", o->target_device_serial, + o->tracker_device_serial); + // drops the target device from the list, but keeps the tracker + // a tracker could be attached to multiple targets with different names + xdevs[target_idx] = multi; + } else { + P_ERROR(p, "Failed to create tracking override multi device"); + } + } +} + + +/* + * + * Member functions. + * + */ + +static int +p_probe(struct xrt_prober *xp) +{ + XRT_TRACE_MARKER(); + struct prober *p = (struct prober *)xp; - enum p_active_config active; + XRT_MAYBE_UNUSED int ret = 0; + + // Free old list first. + teardown_devices(p); + +#ifdef XRT_HAVE_LIBUDEV + ret = p_udev_probe(p); + if (ret != 0) { + P_ERROR(p, "Failed to enumerate udev devices\n"); + return -1; + } +#endif + +#ifdef XRT_HAVE_LIBUSB + ret = p_libusb_probe(p); + if (ret != 0) { + P_ERROR(p, "Failed to enumerate libusb devices\n"); + return -1; + } +#endif + +#ifdef XRT_HAVE_LIBUVC + ret = p_libuvc_probe(p); + if (ret != 0) { + P_ERROR(p, "Failed to enumerate libuvc devices\n"); + return -1; + } +#endif + + return 0; +} + +static int +p_dump(struct xrt_prober *xp) +{ + XRT_TRACE_MARKER(); + + struct prober *p = (struct prober *)xp; + XRT_MAYBE_UNUSED ssize_t k = 0; + XRT_MAYBE_UNUSED size_t j = 0; + + for (size_t i = 0; i < p->num_devices; i++) { + struct prober_device *pdev = &p->devices[i]; + p_dump_device(p, pdev, (int)i); + } + + return 0; +} + +static int +p_select_device(struct xrt_prober *xp, struct xrt_device **xdevs, size_t num_xdevs) +{ + XRT_TRACE_MARKER(); + + struct prober *p = (struct prober *)xp; + enum u_config_json_active_config active; bool have_hmd = false; - p_json_get_active(p, &active); + u_config_json_get_active(&p->json, &active); switch (active) { - case P_ACTIVE_CONFIG_NONE: - case P_ACTIVE_CONFIG_TRACKING: + case U_ACTIVE_CONFIG_NONE: + case U_ACTIVE_CONFIG_TRACKING: add_from_devices(p, xdevs, num_xdevs, &have_hmd); add_from_auto_probers(p, xdevs, num_xdevs, &have_hmd); break; - case P_ACTIVE_CONFIG_REMOTE: add_from_remote(p, xdevs, num_xdevs, &have_hmd); break; + case U_ACTIVE_CONFIG_REMOTE: add_from_remote(p, xdevs, num_xdevs, &have_hmd); break; default: assert(false); } @@ -677,6 +901,15 @@ select_device(struct xrt_prober *xp, struct xrt_device **xdevs, size_t num_xdevs break; } + struct xrt_tracking_override overrides[XRT_MAX_TRACKING_OVERRIDES]; + size_t num_overrides = 0; + if (u_config_json_get_tracking_overrides(&p->json, overrides, &num_overrides)) { + for (size_t i = 0; i < num_overrides; i++) { + struct xrt_tracking_override *o = &overrides[i]; + apply_tracking_override(p, xdevs, num_xdevs, o); + } + } + if (have_hmd) { P_DEBUG(p, "Found HMD! '%s'", xdevs[0]->str); return 0; @@ -699,11 +932,13 @@ select_device(struct xrt_prober *xp, struct xrt_device **xdevs, size_t num_xdevs } static int -open_hid_interface(struct xrt_prober *xp, - struct xrt_prober_device *xpdev, - int interface, - struct os_hid_device **out_hid_dev) +p_open_hid_interface(struct xrt_prober *xp, + struct xrt_prober_device *xpdev, + int interface, + struct os_hid_device **out_hid_dev) { + XRT_TRACE_MARKER(); + struct prober_device *pdev = (struct prober_device *)xpdev; int ret; @@ -732,20 +967,29 @@ open_hid_interface(struct xrt_prober *xp, return -1; } -DEBUG_GET_ONCE_OPTION(vf_path, "VF_PATH", NULL) - static int -open_video_device(struct xrt_prober *xp, - struct xrt_prober_device *xpdev, - struct xrt_frame_context *xfctx, - struct xrt_fs **out_xfs) +p_open_video_device(struct xrt_prober *xp, + struct xrt_prober_device *xpdev, + struct xrt_frame_context *xfctx, + struct xrt_fs **out_xfs) { + XRT_TRACE_MARKER(); + XRT_MAYBE_UNUSED struct prober_device *pdev = (struct prober_device *)xpdev; -#if defined(XRT_HAVE_VF) +#if defined(XRT_BUILD_DRIVER_EUROC) + // TODO: If both VF_PATH and EUROC_PATH are set, VF will be ignored on calibration + const char *euroc_path = debug_get_option_euroc_path(); + if (euroc_path != NULL) { + *out_xfs = euroc_player_create(xfctx, euroc_path); // Euroc will exit if it can't be created + return 0; + } +#endif + +#if defined(XRT_BUILD_DRIVER_VF) const char *path = debug_get_option_vf_path(); if (path != NULL) { - struct xrt_fs *xfs = vf_fs_create(xfctx, path); + struct xrt_fs *xfs = vf_fs_open_file(xfctx, path); if (xfs) { *out_xfs = xfs; return 0; @@ -772,7 +1016,7 @@ open_video_device(struct xrt_prober *xp, } static int -list_video_devices(struct xrt_prober *xp, xrt_prober_list_video_cb cb, void *ptr) +p_list_video_devices(struct xrt_prober *xp, xrt_prober_list_video_cb cb, void *ptr) { struct prober *p = (struct prober *)xp; @@ -781,6 +1025,11 @@ list_video_devices(struct xrt_prober *xp, xrt_prober_list_video_cb cb, void *ptr cb(xp, NULL, "Video File", "Collabora", path, ptr); } + path = debug_get_option_euroc_path(); + if (path != NULL) { + cb(xp, NULL, "Euroc Dataset", "Collabora", path, ptr); + } + // Loop over all devices and find video devices. for (size_t i = 0; i < p->num_devices; i++) { struct prober_device *pdev = &p->devices[i]; @@ -808,31 +1057,61 @@ list_video_devices(struct xrt_prober *xp, xrt_prober_list_video_cb cb, void *ptr } static int -get_string_descriptor(struct xrt_prober *xp, - struct xrt_prober_device *xpdev, - enum xrt_prober_string which_string, - unsigned char *buffer, - int length) +p_get_entries(struct xrt_prober *xp, + size_t *out_num_entries, + struct xrt_prober_entry ***out_entries, + struct xrt_auto_prober ***out_auto_probers) { + XRT_TRACE_MARKER(); + + struct prober *p = (struct prober *)xp; + *out_num_entries = p->num_entries; + *out_entries = p->entries; + *out_auto_probers = p->auto_probers; + + return 0; +} + +static int +p_get_string_descriptor(struct xrt_prober *xp, + struct xrt_prober_device *xpdev, + enum xrt_prober_string which_string, + unsigned char *buffer, + size_t max_length) +{ + XRT_TRACE_MARKER(); + XRT_MAYBE_UNUSED struct prober *p = (struct prober *)xp; XRT_MAYBE_UNUSED struct prober_device *pdev = (struct prober_device *)xpdev; XRT_MAYBE_UNUSED int ret; #ifdef XRT_HAVE_LIBUSB - if (pdev->usb.dev != NULL) { - ret = p_libusb_get_string_descriptor(p, pdev, which_string, buffer, length); + if (pdev->base.bus == XRT_BUS_TYPE_USB && pdev->usb.dev != NULL) { + ret = p_libusb_get_string_descriptor(p, pdev, which_string, buffer, max_length); if (ret >= 0) { return ret; } } #endif + if (pdev->base.bus == XRT_BUS_TYPE_BLUETOOTH && which_string == XRT_PROBER_STRING_SERIAL_NUMBER) { + union { + uint8_t arr[8]; + uint64_t v; + } u; + u.v = pdev->bluetooth.id; + return snprintf((char *)buffer, max_length, "%02X:%02X:%02X:%02X:%02X:%02X", u.arr[5], u.arr[4], + u.arr[3], u.arr[2], u.arr[1], u.arr[0]); + } + //! @todo add more backends //! @todo make this unicode (utf-16)? utf-8 would be better... return 0; } static bool -can_open(struct xrt_prober *xp, struct xrt_prober_device *xpdev) +p_can_open(struct xrt_prober *xp, struct xrt_prober_device *xpdev) { + XRT_TRACE_MARKER(); + XRT_MAYBE_UNUSED struct prober *p = (struct prober *)xp; XRT_MAYBE_UNUSED struct prober_device *pdev = (struct prober_device *)xpdev; #ifdef XRT_HAVE_LIBUSB @@ -844,10 +1123,11 @@ can_open(struct xrt_prober *xp, struct xrt_prober_device *xpdev) return false; } - static void -destroy(struct xrt_prober **xp) +p_destroy(struct xrt_prober **xp) { + XRT_TRACE_MARKER(); + struct prober *p = (struct prober *)*xp; if (p == NULL) { return; diff --git a/src/xrt/state_trackers/prober/p_prober.h b/src/xrt/state_trackers/prober/p_prober.h index 4c3547714..883676cd4 100644 --- a/src/xrt/state_trackers/prober/p_prober.h +++ b/src/xrt/state_trackers/prober/p_prober.h @@ -16,6 +16,7 @@ #include "xrt/xrt_settings.h" #include "util/u_logging.h" +#include "util/u_config_json.h" #ifdef XRT_HAVE_LIBUSB #include @@ -41,18 +42,6 @@ #define P_WARN(d, ...) U_LOG_IFL_W(d->ll, __VA_ARGS__) #define P_ERROR(d, ...) U_LOG_IFL_E(d->ll, __VA_ARGS__) -#define MAX_AUTO_PROBERS 8 - -/*! - * What config is currently active in the config file. - */ -enum p_active_config -{ - P_ACTIVE_CONFIG_NONE = 0, - P_ACTIVE_CONFIG_TRACKING = 1, - P_ACTIVE_CONFIG_REMOTE = 2, -}; - #ifdef XRT_OS_LINUX /*! * A hidraw interface that a @ref prober_device exposes. @@ -133,13 +122,7 @@ struct prober struct xrt_prober_entry_lists *lists; - struct - { - //! For error reporting, was it loaded but not parsed? - bool file_loaded; - - cJSON *root; - } json; + struct u_config_json json; #ifdef XRT_HAVE_LIBUSB struct @@ -167,6 +150,10 @@ struct prober size_t num_entries; struct xrt_prober_entry **entries; + // must not be accessed after freeing json + size_t num_disabled_drivers; + char **disabled_drivers; + enum u_logging_level ll; }; @@ -177,40 +164,6 @@ struct prober * */ -/*! - * Load the JSON config file. - * - * @public @memberof prober - */ -void -p_json_open_or_create_main_file(struct prober *p); - -/*! - * Read from the JSON loaded json config file and returns the active config, - * can be overridden by `P_OVERRIDE_ACTIVE_CONFIG` envirmental variable. - * - * @public @memberof prober - */ -void -p_json_get_active(struct prober *p, enum p_active_config *out_active); - -/*! - * Extract tracking settings from the JSON. - * - * @public @memberof prober - * @relatesalso xrt_settings_tracking - */ -bool -p_json_get_tracking_settings(struct prober *p, struct xrt_settings_tracking *s); - -/*! - * Extract remote settings from the JSON. - * - * @public @memberof prober - */ -bool -p_json_get_remote_port(struct prober *p, int *out_port); - /*! * Dump the given device to stdout. * @@ -249,7 +202,7 @@ p_dev_get_bluetooth_dev( * Init the tracking factory. * * @private @memberof prober - * @relatesalso xrt_tracking_factory + * @see xrt_tracking_factory */ int p_tracking_init(struct prober *p); @@ -258,7 +211,7 @@ p_tracking_init(struct prober *p); * Teardown the tracking factory. * * @private @memberof prober - * @relatesalso xrt_tracking_factory + * @see xrt_tracking_factory */ void p_tracking_teardown(struct prober *p); diff --git a/src/xrt/state_trackers/prober/p_tracking.c b/src/xrt/state_trackers/prober/p_tracking.c index a6479dbf5..1e7b14873 100644 --- a/src/xrt/state_trackers/prober/p_tracking.c +++ b/src/xrt/state_trackers/prober/p_tracking.c @@ -20,11 +20,16 @@ #include "util/u_var.h" #include "util/u_misc.h" #include "util/u_sink.h" +#include "util/u_config_json.h" #include "p_prober.h" #include #include +#ifdef XRT_BUILD_DRIVER_EUROC +#include "util/u_debug.h" +DEBUG_GET_ONCE_OPTION(euroc_path, "EUROC_PATH", NULL) +#endif /* * @@ -77,6 +82,12 @@ struct p_factory //! Pre-created psvr trackers. struct xrt_tracked_psvr *xtvr; + + //! Have we handed out the slam tracker. + bool started_xts; + + //! Pre-create SLAM tracker. + struct xrt_tracked_slam *xts; #endif // Frameserver. @@ -134,10 +145,8 @@ p_factory_ensure_frameserver(struct p_factory *fact) // We have no tried the settings. fact->tried_settings = true; - if (!p_json_get_tracking_settings(fact->p, &fact->settings)) { - U_LOG_E( - "Could not setup PSVR and/or PSMV tracking, " - "see above."); + if (!u_config_json_get_tracking_settings(&fact->p->json, &fact->settings)) { + U_LOG_I("PSVR and/or PSMV tracking is not set up, see above."); return; } @@ -325,6 +334,44 @@ p_factory_create_tracked_hand(struct xrt_tracking_factory *xfact, #endif } +static int +p_factory_create_tracked_slam(struct xrt_tracking_factory *xfact, + struct xrt_device *xdev, + struct xrt_tracked_slam **out_xts) +{ +#ifdef XRT_HAVE_SLAM + struct p_factory *fact = p_factory(xfact); + + struct xrt_tracked_slam *xts = NULL; + +#ifdef XRT_BUILD_DRIVER_EUROC + if (debug_get_option_euroc_path() != NULL) { + // The euroc slam tracker was already created on p_tracking_init because the + // euroc player is not a device so it needs to be started from somewhere + goto end; + } +#endif + +end: + + if (!fact->started_xts) { + xts = fact->xts; + } + + if (xts == NULL) { + return -1; + } + + fact->started_xts = true; + t_slam_start(xts); + *out_xts = xts; + + return 0; +#else + return -1; +#endif +} + /* * * "Exported" prober functions. @@ -340,6 +387,7 @@ p_tracking_init(struct prober *p) fact->base.create_tracked_psmv = p_factory_create_tracked_psmv; fact->base.create_tracked_psvr = p_factory_create_tracked_psvr; fact->base.create_tracked_hand = p_factory_create_tracked_hand; + fact->base.create_tracked_slam = p_factory_create_tracked_slam; fact->origin.type = XRT_TRACKING_TYPE_RGB; fact->origin.offset.orientation.y = 1.0f; fact->origin.offset.position.z = -2.0f; @@ -354,6 +402,27 @@ p_tracking_init(struct prober *p) // Finally set us as the tracking factory. p->base.tracking = &fact->base; +#ifdef XRT_BUILD_DRIVER_EUROC + if (debug_get_option_euroc_path() != NULL) { + struct xrt_slam_sinks empty_sinks = {0}; + struct xrt_slam_sinks *sinks = &empty_sinks; + + // fact->xfs *will* be an euroc frame server after open, because of prober open_video_device + xrt_prober_open_video_device(&fact->p->base, NULL, &fact->xfctx, &fact->xfs); + +#ifdef XRT_HAVE_SLAM + int ret = t_slam_create(&fact->xfctx, &fact->xts, &sinks); + if (ret != 0) { + U_LOG_W("Unable to initialize SLAM tracking, the Euroc driver will not be tracked"); + } +#else + U_LOG_W("SLAM tracking support is disabled, the Euroc driver will not be tracked"); +#endif + + xrt_fs_slam_stream_start(fact->xfs, sinks); + } +#endif + return 0; } diff --git a/src/xrt/state_trackers/steamvr_drv/meson.build b/src/xrt/state_trackers/steamvr_drv/meson.build index 17aa5806b..773608d1e 100644 --- a/src/xrt/state_trackers/steamvr_drv/meson.build +++ b/src/xrt/state_trackers/steamvr_drv/meson.build @@ -14,7 +14,7 @@ lib_st_ovrd = static_library( st_include, # Sigh debian meson requires this. xrt_include, ], - dependencies: aux_util, + dependencies: [aux_util, aux_generated_bindings], c_args: compile_args, cpp_args: compile_args, ) diff --git a/src/xrt/state_trackers/steamvr_drv/ovrd_driver.cpp b/src/xrt/state_trackers/steamvr_drv/ovrd_driver.cpp index 1fa4274e3..42d773bd3 100644 --- a/src/xrt/state_trackers/steamvr_drv/ovrd_driver.cpp +++ b/src/xrt/state_trackers/steamvr_drv/ovrd_driver.cpp @@ -42,6 +42,8 @@ DEBUG_GET_ONCE_BOOL_OPTION(emulate_index_controller, "STEAMVR_EMULATE_INDEX_CONT DEBUG_GET_ONCE_NUM_OPTION(scale_percentage, "XRT_COMPOSITOR_SCALE_PERCENTAGE", 140) +#define MODELNUM_LEN (XRT_DEVICE_NAME_LEN + 9) // "[Monado] " + //#define DUMP_POSE //#define DUMP_POSE_CONTROLLERS @@ -148,16 +150,8 @@ public: m_unObjectId = vr::k_unTrackedDeviceIndexInvalid; m_pose = {}; - // append xrt_hand because SteamVR serial must be unique - std::stringstream ss; - ss << "[Monado] " << xdev->str << " " << hand; - std::string name = ss.str(); - - strncpy(m_sSerialNumber, name.c_str(), XRT_DEVICE_NAME_LEN); - strncpy(m_sModelNumber, name.c_str(), XRT_DEVICE_NAME_LEN); - - strncpy(m_sSerialNumber, name.c_str(), XRT_DEVICE_NAME_LEN); - strncpy(m_sModelNumber, name.c_str(), XRT_DEVICE_NAME_LEN); + snprintf(m_sModelNumber, MODELNUM_LEN, "[Monado] %s", xdev->str); + strncpy(m_sSerialNumber, xdev->serial, XRT_DEVICE_NAME_LEN); switch (this->m_xdev->name) { case XRT_DEVICE_INDEX_CONTROLLER: @@ -172,6 +166,14 @@ public: "right"; } break; + case XRT_DEVICE_TOUCH_CONTROLLER: + if (hand == XRT_HAND_LEFT) { + m_render_model = "oculus_cv1_controller_left"; + } + if (hand == XRT_HAND_RIGHT) { + m_render_model = "oculus_cv1_controller_right"; + } + break; case XRT_DEVICE_VIVE_WAND: m_render_model = "vr_controller_vive_1_5"; break; case XRT_DEVICE_VIVE_TRACKER_GEN1: case XRT_DEVICE_VIVE_TRACKER_GEN2: m_render_model = "{htc}vr_tracker_vive_1_0"; break; @@ -344,6 +346,7 @@ public: case XRT_DEVICE_XBOX_CONTROLLER: break; // TODO case XRT_DEVICE_VIVE_TRACKER_GEN1: break; // TODO case XRT_DEVICE_VIVE_TRACKER_GEN2: break; // TODO + case XRT_DEVICE_REALSENSE: break; case XRT_DEVICE_HAND_INTERACTION: break; // there is no hardware case XRT_DEVICE_GO_CONTROLLER: break; // hardware has no haptics @@ -486,6 +489,11 @@ public: struct profile_template *p = get_profile_template(m_xdev->name); + if (p == NULL) { + ovrd_log("Monado device has unknown profile: %d\n", m_xdev->name); + return vr::VRInitError_Unknown; + } + m_input_profile = std::string("{monado}/input/") + std::string(p->steamvr_input_profile_path); m_controller_type = p->steamvr_controller_type; } @@ -571,6 +579,8 @@ public: grip_name = XRT_INPUT_DAYDREAM_POSE; } else if (m_xdev->name == XRT_DEVICE_HYDRA) { grip_name = XRT_INPUT_HYDRA_POSE; + } else if (m_xdev->name == XRT_DEVICE_TOUCH_CONTROLLER) { + grip_name = XRT_INPUT_TOUCH_GRIP_POSE; } else { ovrd_log("Unhandled device name %u\n", m_xdev->name); grip_name = XRT_INPUT_GENERIC_HEAD_POSE; // ??? @@ -703,7 +713,7 @@ public: private: char m_sSerialNumber[XRT_DEVICE_NAME_LEN]; - char m_sModelNumber[XRT_DEVICE_NAME_LEN]; + char m_sModelNumber[MODELNUM_LEN]; const char *m_controller_type = NULL; @@ -1226,7 +1236,7 @@ CServerDriver_Monado::HandleHapticEvent(vr::VREvent_t *event) union xrt_output_value out; out.vibration.amplitude = amp; if (duration > 0.00001) { - out.vibration.duration = duration * 1000. * 1000. * 1000.; + out.vibration.duration = (time_duration_ns)(duration * 1000.f * 1000.f * 1000.f); } else { out.vibration.duration = XRT_MIN_HAPTIC_DURATION; } diff --git a/src/xrt/targets/android_common/src/main/AndroidManifest.xml b/src/xrt/targets/android_common/src/main/AndroidManifest.xml index 1d758e8fd..2b1f76387 100644 --- a/src/xrt/targets/android_common/src/main/AndroidManifest.xml +++ b/src/xrt/targets/android_common/src/main/AndroidManifest.xml @@ -6,6 +6,9 @@ SPDX-License-Identifier: BSL-1.0 --> + + + (R.id.btnLaunchDisplayOverOtherAppsSettings) + .setOnClickListener { launchDisplayOverOtherAppsSettings() } + return view + } + + private fun updateStatus(view: View?) { + displayOverOtherAppsEnabled = Settings.canDrawOverlays(requireContext()) + val tv = view!!.findViewById(R.id.textDisplayOverOtherAppsStatus) + // Combining format with html style tag might have problem. See + // https://developer.android.com/guide/topics/resources/string-resource.html#StylingWithHTML + val msg = getString( + R.string.msg_display_over_other_apps, + if (displayOverOtherAppsEnabled) getString(R.string.enabled) else getString(R.string.disabled) + ) + tv.text = Html.fromHtml(msg, Html.FROM_HTML_MODE_LEGACY) + } + + private fun launchDisplayOverOtherAppsSettings() { + // Since Android 11, framework ignores the uri and takes user to the top-level settings. + // See https://developer.android.com/about/versions/11/privacy/permissions#system-alert + // for detail. + val intent = Intent( + Settings.ACTION_MANAGE_OVERLAY_PERMISSION, + Uri.parse("package:" + context!!.packageName) + ) + startActivityForResult(intent, REQUEST_CODE_DISPLAY_OVER_OTHER_APPS) + } + + override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { + // resultCode is always Activity.RESULT_CANCELED + if (requestCode != REQUEST_CODE_DISPLAY_OVER_OTHER_APPS) { + return + } + + if (isRuntimeServiceRunning && + displayOverOtherAppsEnabled != Settings.canDrawOverlays(requireContext()) + ) { + showRestartDialog() + } else { + updateStatus(view) + } + } + + @Suppress("DEPRECATION") + private val isRuntimeServiceRunning: Boolean + get() { + var running = false + val am = requireContext().getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager + for (service in am.getRunningServices(Int.MAX_VALUE)) { + if (service.pid == Process.myPid()) { + running = true + break + } + } + return running + } + + private fun showRestartDialog() { + val dialog: DialogFragment = RestartRuntimeDialogFragment.newInstance( + getString(R.string.msg_display_over_other_apps_changed) + ) + dialog.show(parentFragmentManager, null) + } + + companion object { + private const val REQUEST_CODE_DISPLAY_OVER_OTHER_APPS = 1000 + } +} diff --git a/src/xrt/targets/android_common/src/main/java/org/freedesktop/monado/android_common/RestartRuntimeDialogFragment.kt b/src/xrt/targets/android_common/src/main/java/org/freedesktop/monado/android_common/RestartRuntimeDialogFragment.kt new file mode 100644 index 000000000..844135588 --- /dev/null +++ b/src/xrt/targets/android_common/src/main/java/org/freedesktop/monado/android_common/RestartRuntimeDialogFragment.kt @@ -0,0 +1,61 @@ +// Copyright 2021, Qualcomm Innovation Center, Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Fragment to display the reason of runtime restart. + * @author Jarvis Huang + */ +package org.freedesktop.monado.android_common + +import android.app.AlarmManager +import android.app.Dialog +import android.app.PendingIntent +import android.content.Context +import android.content.DialogInterface +import android.content.Intent +import android.os.Bundle +import android.os.Process +import androidx.appcompat.app.AlertDialog +import androidx.fragment.app.DialogFragment + +class RestartRuntimeDialogFragment : DialogFragment() { + + override fun onCreateDialog(savedInstanceState: Bundle?): Dialog { + val message = arguments!!.getString(ARGS_KEY_MESSAGE) + val builder = AlertDialog.Builder(requireActivity()) + builder.setMessage(message) + .setCancelable(false) + .setPositiveButton(R.string.restart) { _: DialogInterface?, _: Int -> + delayRestart(DELAY_RESTART_DURATION) + //! @todo elegant way to stop service? A bounded service might be restarted by + // framework automatically. + Process.killProcess(Process.myPid()) + } + return builder.create() + } + + private fun delayRestart(delayMillis: Long) { + val intent = Intent(requireContext(), AboutActivity::class.java) + val pendingIntent = PendingIntent.getActivity( + requireContext(), REQUEST_CODE, + intent, PendingIntent.FLAG_CANCEL_CURRENT + ) + val am = requireContext().getSystemService(Context.ALARM_SERVICE) as AlarmManager + am.setExact(AlarmManager.RTC, System.currentTimeMillis() + delayMillis, pendingIntent) + } + + companion object { + private const val ARGS_KEY_MESSAGE = "message" + private const val REQUEST_CODE = 2000 + private const val DELAY_RESTART_DURATION: Long = 200 + + @JvmStatic + fun newInstance(msg: String): RestartRuntimeDialogFragment { + val fragment = RestartRuntimeDialogFragment() + val args = Bundle() + args.putString(ARGS_KEY_MESSAGE, msg) + fragment.arguments = args + return fragment + } + } +} diff --git a/src/xrt/targets/android_common/src/main/res/layout/activity_about.xml b/src/xrt/targets/android_common/src/main/res/layout/activity_about.xml index bfe1f14b9..387060ffa 100644 --- a/src/xrt/targets/android_common/src/main/res/layout/activity_about.xml +++ b/src/xrt/targets/android_common/src/main/res/layout/activity_about.xml @@ -32,7 +32,7 @@ app:layout_constraintEnd_toEndOf="parent" app:layout_constraintHorizontal_bias="0.5" app:layout_constraintStart_toStartOf="parent" - app:layout_constraintTop_toBottomOf="@+id/imageView" /> + app:layout_constraintTop_toBottomOf="@id/imageView" /> + app:layout_constraintTop_toBottomOf="@id/versionView" /> + app:layout_constraintTop_toBottomOf="@id/textName" />