diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 2722c68a9..b7b4aebe1 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -101,7 +101,7 @@ void MemoryManager::Free(PAddr phys_addr, size_t size) {
         }
     }
     for (const auto& [addr, size] : remove_list) {
-        UnmapMemory(addr, size);
+        UnmapMemoryImpl(addr, size);
     }
 
     // Mark region as free and attempt to coalesce it with neighbours.
@@ -124,7 +124,7 @@ int MemoryManager::Reserve(void** out_addr, VAddr virtual_addr, size_t size, Mem
         const auto& vma = FindVMA(mapped_addr)->second;
         // If the VMA is mapped, unmap the region first.
         if (vma.IsMapped()) {
-            UnmapMemory(mapped_addr, size);
+            UnmapMemoryImpl(mapped_addr, size);
         }
         const size_t remaining_size = vma.base + vma.size - mapped_addr;
         ASSERT_MSG(vma.type == VMAType::Free && remaining_size >= size);
@@ -233,7 +233,10 @@ int MemoryManager::MapFile(void** out_addr, VAddr virtual_addr, size_t size, Mem
 
 void MemoryManager::UnmapMemory(VAddr virtual_addr, size_t size) {
     std::scoped_lock lk{mutex};
+    UnmapMemoryImpl(virtual_addr, size);
+}
 
+void MemoryManager::UnmapMemoryImpl(VAddr virtual_addr, size_t size) {
     const auto it = FindVMA(virtual_addr);
     const auto& vma_base = it->second;
     ASSERT_MSG(vma_base.Contains(virtual_addr, size),
diff --git a/src/core/memory.h b/src/core/memory.h
index c8638626a..b7edee33a 100644
--- a/src/core/memory.h
+++ b/src/core/memory.h
@@ -214,11 +214,13 @@ private:
 
     DMemHandle Split(DMemHandle dmem_handle, size_t offset_in_area);
 
+    void UnmapMemoryImpl(VAddr virtual_addr, size_t size);
+
 private:
     AddressSpace impl;
     DMemMap dmem_map;
     VMAMap vma_map;
-    std::recursive_mutex mutex;
+    std::mutex mutex;
     size_t total_direct_size{};
     size_t total_flexible_size{};
     size_t flexible_usage{};
diff --git a/src/video_core/buffer_cache/buffer_cache.cpp b/src/video_core/buffer_cache/buffer_cache.cpp
index 0151f2c13..71228786e 100644
--- a/src/video_core/buffer_cache/buffer_cache.cpp
+++ b/src/video_core/buffer_cache/buffer_cache.cpp
@@ -230,7 +230,6 @@ u32 BufferCache::BindIndexBuffer(bool& is_indexed, u32 index_offset) {
 
 std::pair<Buffer*, u32> BufferCache::ObtainBuffer(VAddr device_addr, u32 size, bool is_written,
                                                   bool is_texel_buffer) {
-    std::scoped_lock lk{mutex};
     static constexpr u64 StreamThreshold = CACHING_PAGESIZE;
     const bool is_gpu_dirty = memory_tracker.IsRegionGpuModified(device_addr, size);
     if (!is_written && !is_texel_buffer && size <= StreamThreshold && !is_gpu_dirty) {
@@ -461,6 +460,7 @@ void BufferCache::ChangeRegister(BufferId buffer_id) {
 }
 
 bool BufferCache::SynchronizeBuffer(Buffer& buffer, VAddr device_addr, u32 size) {
+    std::scoped_lock lk{mutex};
     boost::container::small_vector<vk::BufferCopy, 4> copies;
     u64 total_size_bytes = 0;
     u64 largest_copy = 0;
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index a07470b8e..b9002cea2 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -126,7 +126,7 @@ private:
     PageManager& tracker;
     StreamBuffer staging_buffer;
     StreamBuffer stream_buffer;
-    std::recursive_mutex mutex;
+    std::mutex mutex;
     Common::SlotVector<Buffer> slot_buffers;
     MemoryTracker memory_tracker;
     PageTable page_table;