diff --git a/CompilerFlags.cmake b/CompilerFlags.cmake
index 55647a979..40a1c97b4 100644
--- a/CompilerFlags.cmake
+++ b/CompilerFlags.cmake
@@ -1,7 +1,13 @@
 # Copyright 2018-2023, Collabora, Ltd.
 # SPDX-License-Identifier: BSL-1.0
 
-if(NOT MSVC)
+
+# Target used for applying more aggressive optimizations to math-heavy code
+add_library(xrt-optimized-math INTERFACE)
+
+if(MSVC)
+	target_compile_options(xrt-optimized-math INTERFACE $<IF:$<CONFIG:Debug>,/O2,/O3>)
+else()
 	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pedantic -Wall -Wextra -Wno-unused-parameter")
 	set(CMAKE_C_FLAGS
 	    "${CMAKE_C_FLAGS} -Werror-implicit-function-declaration -Werror=incompatible-pointer-types"
@@ -15,6 +21,8 @@ if(NOT MSVC)
 		set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse2 -mfpmath=sse")
 		set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse2 -mfpmath=sse")
 	endif()
+
+	target_compile_options(xrt-optimized-math INTERFACE $<IF:$<CONFIG:Debug>,-O2,-O3>)
 endif()
 
 if(NOT WIN32)
diff --git a/src/xrt/auxiliary/math/CMakeLists.txt b/src/xrt/auxiliary/math/CMakeLists.txt
index a5afa9045..e7b4f83b0 100644
--- a/src/xrt/auxiliary/math/CMakeLists.txt
+++ b/src/xrt/auxiliary/math/CMakeLists.txt
@@ -37,5 +37,9 @@ add_library(
 	m_vec2.h
 	m_vec3.h
 	)
-target_link_libraries(aux_math PUBLIC aux-includes aux_util)
+target_link_libraries(
+	aux_math
+	PUBLIC aux-includes aux_util
+	PRIVATE xrt-optimized-math
+	)
 target_include_directories(aux_math SYSTEM PRIVATE ${EIGEN3_INCLUDE_DIR})
diff --git a/src/xrt/auxiliary/tracking/CMakeLists.txt b/src/xrt/auxiliary/tracking/CMakeLists.txt
index fb54d1f5f..d4e131b8d 100644
--- a/src/xrt/auxiliary/tracking/CMakeLists.txt
+++ b/src/xrt/auxiliary/tracking/CMakeLists.txt
@@ -14,7 +14,12 @@ add_library(
 target_link_libraries(
 	aux_tracking
 	PUBLIC aux-includes xrt-external-cjson
-	PRIVATE aux_math aux_util xrt-external-flexkalman xrt-external-hungarian
+	PRIVATE
+		aux_math
+		aux_util
+		xrt-external-flexkalman
+		xrt-external-hungarian
+		xrt-optimized-math
 	)
 target_include_directories(aux_tracking SYSTEM PRIVATE ${EIGEN3_INCLUDE_DIR})
 
diff --git a/src/xrt/tracking/hand/mercury/CMakeLists.txt b/src/xrt/tracking/hand/mercury/CMakeLists.txt
index 6d7012aef..f8354f710 100644
--- a/src/xrt/tracking/hand/mercury/CMakeLists.txt
+++ b/src/xrt/tracking/hand/mercury/CMakeLists.txt
@@ -1,10 +1,11 @@
-# Copyright 2019-2022, Collabora, Ltd.
+# Copyright 2019-2023, Collabora, Ltd.
 # SPDX-License-Identifier: BSL-1.0
 
 # Mercury hand tracking library!
 
 add_subdirectory(kine_lm)
 
+# t_ht_mercury_model
 add_library(t_ht_mercury_model STATIC hg_model.cpp)
 
 target_link_libraries(t_ht_mercury_model PRIVATE aux_math aux_tracking aux_os aux_util)
@@ -20,10 +21,12 @@ target_link_libraries(
 		aux_tracking
 		aux_os
 		aux_util
+		xrt-optimized-math
 		${OpenCV_LIBRARIES}
 		ONNXRuntime::ONNXRuntime
 	)
 
+# t_ht_mercury_distorter
 add_library(t_ht_mercury_distorter STATIC hg_image_distorter.cpp)
 
 target_link_libraries(t_ht_mercury_distorter PRIVATE aux_math aux_tracking aux_os aux_util)
@@ -39,10 +42,12 @@ target_link_libraries(
 		aux_tracking
 		aux_os
 		aux_util
+		xrt-optimized-math
 		${OpenCV_LIBRARIES}
 		ONNXRuntime::ONNXRuntime # no, wrong
 	)
 
+# t_ht_mercury
 add_library(t_ht_mercury STATIC hg_sync.cpp hg_sync.hpp hg_interface.h kine_common.hpp)
 
 target_link_libraries(
@@ -57,6 +62,7 @@ target_link_libraries(
 		t_ht_mercury_kine_lm
 		t_ht_mercury_model
 		t_ht_mercury_distorter
+		xrt-optimized-math
 		${OpenCV_LIBRARIES}
 	)
 
diff --git a/src/xrt/tracking/hand/mercury/kine_lm/CMakeLists.txt b/src/xrt/tracking/hand/mercury/kine_lm/CMakeLists.txt
index 2b6e37010..0a9181fca 100644
--- a/src/xrt/tracking/hand/mercury/kine_lm/CMakeLists.txt
+++ b/src/xrt/tracking/hand/mercury/kine_lm/CMakeLists.txt
@@ -1,4 +1,4 @@
-# Copyright 2022, Collabora, Ltd.
+# Copyright 2022-2023, Collabora, Ltd.
 # SPDX-License-Identifier: BSL-1.0
 
 add_library(
@@ -14,6 +14,7 @@ target_link_libraries(
 		aux_os
 		aux_util
 		xrt-external-tinyceres
+		xrt-optimized-math
 	)
 
 target_include_directories(t_ht_mercury_kine_lm SYSTEM PRIVATE ${EIGEN3_INCLUDE_DIR})