diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3bbc8177b..b59bbfa12 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -48,7 +48,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
 	pkg_check_modules(XCB xcb xcb-randr)
 
 	find_package(udev REQUIRED)
-	set(BUILD_WITH_LIBUDEV TRUE)
 	set(BUILD_DRIVER_V4L2 TRUE)
 
 	pkg_search_module(WAYLAND wayland-client)
@@ -71,10 +70,15 @@ cmake_dependent_option(BUILD_TARGET_OPENXR "Build OpenXR runtime target" ON "BUI
 # Most users won't touch these.
 mark_as_advanced(BUILD_COMPOSITOR_MAIN BUILD_TARGET_OPENXR)
 
-set(BUILD_WITH_LIBUSB TRUE)
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+	set(BUILD_WITH_LIBUDEV ON)
+else()
+	cmake_dependent_option(BUILD_WITH_LIBUDEV "Enable libudev (used for device probing on Linux)" ON "UDEV_FOUND" OFF)
+endif()
+cmake_dependent_option(BUILD_WITH_LIBUSB "Enable libusb (used for most drivers)" ON "LIBUSB1_FOUND" OFF)
 cmake_dependent_option(BUILD_WITH_JPEG "Enable jpeg code (used for some video drivers)" ON "JPEG_FOUND" OFF)
 cmake_dependent_option(BUILD_WITH_OPENCV "Enable OpenCV backend" ON "OpenCV_FOUND" OFF)
-cmake_dependent_option(BUILD_WITH_LIBUVC "Enable libuvc video driver" ON "LIBUVC_FOUND" OFF)
+cmake_dependent_option(BUILD_WITH_LIBUVC "Enable libuvc video driver" ON "LIBUVC_FOUND AND BUILD_WITH_LIBUSB" OFF)
 cmake_dependent_option(BUILD_WITH_FFMPEG "Enable ffmpeg testing video driver" ON "FFMPEG_FOUND" OFF)
 cmake_dependent_option(BUILD_WITH_PSVR "Enable PSVR HMD driver" ON "HIDAPI_FOUND" OFF)
 cmake_dependent_option(BUILD_WITH_RS "Enable RealSense device driver" ON "realsense2_FOUND" OFF)
diff --git a/src/xrt/targets/CMakeLists.txt b/src/xrt/targets/CMakeLists.txt
index d9d2d99e7..d3f0ddbfc 100644
--- a/src/xrt/targets/CMakeLists.txt
+++ b/src/xrt/targets/CMakeLists.txt
@@ -7,8 +7,6 @@
 
 
 # Set up these two lists for use across multiple targets.
-set(DRIVER_OBJECTS)
-set(DRIVER_LIBRARIES)
 
 list(APPEND DRIVER_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
 
@@ -16,6 +14,14 @@ if(BUILD_WITH_LIBUSB)
 	list(APPEND DRIVER_LIBRARIES ${LIBUSB1_LIBRARIES})
 endif()
 
+if(BUILD_WITH_LIBUVC)
+	list(APPEND DRIVER_LIBRARIES ${LIBUVC_LIBRARIES})
+endif()
+
+if(BUILD_WITH_LIBUDEV)
+	list(APPEND DRIVER_LIBRARIES ${UDEV_LIBRARIES})
+endif()
+
 if(BUILD_WITH_JPEG)
 	list(APPEND DRIVER_LIBRARIES ${JPEG_LIBRARIES})
 endif()
diff --git a/src/xrt/targets/cli/CMakeLists.txt b/src/xrt/targets/cli/CMakeLists.txt
index aeb26134a..994bf24d2 100644
--- a/src/xrt/targets/cli/CMakeLists.txt
+++ b/src/xrt/targets/cli/CMakeLists.txt
@@ -35,9 +35,6 @@ set_target_properties(cli PROPERTIES
 	)
 
 target_link_libraries(cli PRIVATE
-	${LIBUSB_LIBRARIES}
-	${LIBUVC_LIBRARIES}
-	${UDEV_LIBRARIES}
 	Threads::Threads
 	)