Disallow concurrent fence waits on Adreno (#7001)

* Disallow concurrent fence waits on Adreno

* Ensure locks are released if exceptions are thrown

src/Ryujinx.Graphics.Vulkan/BackgroundResources.cs

@@ -29,7 +29,14 @@ namespace Ryujinx.Graphics.Vulkan
 
                 lock (queueLock)
                 {
-                    _pool = new CommandBufferPool(_gd.Api, _device, queue, queueLock, _gd.QueueFamilyIndex, isLight: true);
+                    _pool = new CommandBufferPool(
+                        _gd.Api,
+                        _device,
+                        queue,
+                        queueLock,
+                        _gd.QueueFamilyIndex,
+                        _gd.IsConcurrentFenceWaitUnsupported,
+                        isLight: true);
                 }
             }
 

src/Ryujinx.Graphics.Vulkan/CommandBufferPool.cs

@@ -18,6 +18,7 @@ namespace Ryujinx.Graphics.Vulkan
         private readonly Device _device;
         private readonly Queue _queue;
         private readonly object _queueLock;
+        private readonly bool _concurrentFenceWaitUnsupported;
         private readonly CommandPool _pool;
         private readonly Thread _owner;
 
@@ -61,12 +62,20 @@ namespace Ryujinx.Graphics.Vulkan
         private int _queuedCount;
         private int _inUseCount;
 
-        public unsafe CommandBufferPool(Vk api, Device device, Queue queue, object queueLock, uint queueFamilyIndex, bool isLight = false)
+        public unsafe CommandBufferPool(
+            Vk api,
+            Device device,
+            Queue queue,
+            object queueLock,
+            uint queueFamilyIndex,
+            bool concurrentFenceWaitUnsupported,
+            bool isLight = false)
         {
             _api = api;
             _device = device;
             _queue = queue;
             _queueLock = queueLock;
+            _concurrentFenceWaitUnsupported = concurrentFenceWaitUnsupported;
             _owner = Thread.CurrentThread;
 
             var commandPoolCreateInfo = new CommandPoolCreateInfo
@@ -357,7 +366,7 @@ namespace Ryujinx.Graphics.Vulkan
 
             if (refreshFence)
             {
-                entry.Fence = new FenceHolder(_api, _device);
+                entry.Fence = new FenceHolder(_api, _device, _concurrentFenceWaitUnsupported);
             }
             else
             {

src/Ryujinx.Graphics.Vulkan/FenceHolder.cs

@@ -10,12 +10,15 @@ namespace Ryujinx.Graphics.Vulkan
         private readonly Device _device;
         private Fence _fence;
         private int _referenceCount;
+        private int _lock;
+        private readonly bool _concurrentWaitUnsupported;
         private bool _disposed;
 
-        public unsafe FenceHolder(Vk api, Device device)
+        public unsafe FenceHolder(Vk api, Device device, bool concurrentWaitUnsupported)
         {
             _api = api;
             _device = device;
+            _concurrentWaitUnsupported = concurrentWaitUnsupported;
 
             var fenceCreateInfo = new FenceCreateInfo
             {
@@ -47,6 +50,11 @@ namespace Ryujinx.Graphics.Vulkan
             }
             while (Interlocked.CompareExchange(ref _referenceCount, lastValue + 1, lastValue) != lastValue);
 
+            if (_concurrentWaitUnsupported)
+            {
+                AcquireLock();
+            }
+
             fence = _fence;
             return true;
         }
@@ -57,6 +65,16 @@ namespace Ryujinx.Graphics.Vulkan
             return _fence;
         }
 
+        public void PutLock()
+        {
+            Put();
+
+            if (_concurrentWaitUnsupported)
+            {
+                ReleaseLock();
+            }
+        }
+
         public void Put()
         {
             if (Interlocked.Decrement(ref _referenceCount) == 0)
@@ -66,24 +84,67 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
 
+        private void AcquireLock()
+        {
+            while (!TryAcquireLock())
+            {
+                Thread.SpinWait(32);
+            }
+        }
+
+        private bool TryAcquireLock()
+        {
+            return Interlocked.Exchange(ref _lock, 1) == 0;
+        }
+
+        private void ReleaseLock()
+        {
+            Interlocked.Exchange(ref _lock, 0);
+        }
+
         public void Wait()
         {
-            Span<Fence> fences = stackalloc Fence[]
+            if (_concurrentWaitUnsupported)
             {
-                _fence,
+                AcquireLock();
-            };
 
-            FenceHelper.WaitAllIndefinitely(_api, _device, fences);
+                try
+                {
+                    FenceHelper.WaitAllIndefinitely(_api, _device, stackalloc Fence[] { _fence });
+                }
+                finally
+                {
+                    ReleaseLock();
+                }
+            }
+            else
+            {
+                FenceHelper.WaitAllIndefinitely(_api, _device, stackalloc Fence[] { _fence });
+            }
         }
 
         public bool IsSignaled()
         {
-            Span<Fence> fences = stackalloc Fence[]
+            if (_concurrentWaitUnsupported)
             {
-                _fence,
+                if (!TryAcquireLock())
-            };
+                {
+                    return false;
+                }
 
-            return FenceHelper.AllSignaled(_api, _device, fences);
+                try
+                {
+                    return FenceHelper.AllSignaled(_api, _device, stackalloc Fence[] { _fence });
+                }
+                finally
+                {
+                    ReleaseLock();
+                }
+            }
+            else
+            {
+                return FenceHelper.AllSignaled(_api, _device, stackalloc Fence[] { _fence });
+            }
         }
 
         public void Dispose()

src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs

@@ -196,18 +196,23 @@ namespace Ryujinx.Graphics.Vulkan
 
             bool signaled = true;
 
-            if (hasTimeout)
+            try
             {
-                signaled = FenceHelper.AllSignaled(api, device, fences[..fenceCount], timeout);
+                if (hasTimeout)
+                {
+                    signaled = FenceHelper.AllSignaled(api, device, fences[..fenceCount], timeout);
+                }
+                else
+                {
+                    FenceHelper.WaitAllIndefinitely(api, device, fences[..fenceCount]);
+                }
             }
-            else
+            finally
             {
-                FenceHelper.WaitAllIndefinitely(api, device, fences[..fenceCount]);
+                for (int i = 0; i < fenceCount; i++)
-            }
+                {
-
+                    fenceHolders[i].PutLock();
-            for (int i = 0; i < fenceCount; i++)
+                }
-            {
-                fenceHolders[i].Put();
             }
 
             return signaled;

src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs

@@ -90,6 +90,8 @@ namespace Ryujinx.Graphics.Vulkan
         internal bool IsMoltenVk { get; private set; }
         internal bool IsTBDR { get; private set; }
         internal bool IsSharedMemory { get; private set; }
+        internal bool IsConcurrentFenceWaitUnsupported { get; private set; }
+
         public string GpuVendor { get; private set; }
         public string GpuDriver { get; private set; }
         public string GpuRenderer { get; private set; }
@@ -323,6 +325,8 @@ namespace Ryujinx.Graphics.Vulkan
                 Vendor == Vendor.Broadcom ||
                 Vendor == Vendor.ImgTec;
 
+            IsConcurrentFenceWaitUnsupported = Vendor == Vendor.Qualcomm;
+
             GpuVendor = VendorUtils.GetNameFromId(properties.VendorID);
             GpuDriver = hasDriverProperties && !OperatingSystem.IsMacOS() ?
                 VendorUtils.GetFriendlyDriverName(driverProperties.DriverID) : GpuVendor; // Fallback to vendor name if driver is unavailable or on MacOS where vendor is preferred.
@@ -411,7 +415,7 @@ namespace Ryujinx.Graphics.Vulkan
             Api.TryGetDeviceExtension(_instance.Instance, _device, out ExtExternalMemoryHost hostMemoryApi);
             HostMemoryAllocator = new HostMemoryAllocator(MemoryAllocator, Api, hostMemoryApi, _device);
 
-            CommandBufferPool = new CommandBufferPool(Api, _device, Queue, QueueLock, queueFamilyIndex);
+            CommandBufferPool = new CommandBufferPool(Api, _device, Queue, QueueLock, queueFamilyIndex, IsConcurrentFenceWaitUnsupported);
 
             PipelineLayoutCache = new PipelineLayoutCache();