diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index 12461e96e..e01e5142c 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -5,6 +5,8 @@ using Ryujinx.Memory.Tracking; using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory { @@ -65,6 +67,9 @@ namespace Ryujinx.Graphics.Gpu.Memory private readonly Action<ulong, ulong> _loadDelegate; private readonly Action<ulong, ulong> _modifiedDelegate; + private HashSet<MultiRangeBuffer> _virtualDependencies; + private readonly ReaderWriterLockSlim _virtualDependenciesLock; + private int _sequenceNumber; private readonly bool _useGranular; @@ -152,6 +157,8 @@ namespace Ryujinx.Graphics.Gpu.Memory _externalFlushDelegate = new RegionSignal(ExternalFlush); _loadDelegate = new Action<ulong, ulong>(LoadRegion); _modifiedDelegate = new Action<ulong, ulong>(RegionModified); + + _virtualDependenciesLock = new ReaderWriterLockSlim(); } /// <summary> @@ -220,6 +227,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// </remarks> /// <param name="address">Start address of the range to synchronize</param> /// <param name="size">Size in bytes of the range to synchronize</param> + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void SynchronizeMemory(ulong address, ulong size) { if (_useGranular) @@ -239,6 +247,7 @@ namespace Ryujinx.Graphics.Gpu.Memory else { _context.Renderer.SetBufferData(Handle, 0, _physicalMemory.GetSpan(Address, (int)Size)); + CopyToDependantVirtualBuffers(); } _sequenceNumber = _context.SequenceNumber; @@ -460,6 +469,8 @@ namespace Ryujinx.Graphics.Gpu.Memory int offset = (int)(mAddress - Address); _context.Renderer.SetBufferData(Handle, offset, _physicalMemory.GetSpan(mAddress, (int)mSize)); + + CopyToDependantVirtualBuffers(mAddress, mSize); } /// <summary> @@ -520,6 +531,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// <param name="dstOffset">The offset of the destination buffer to copy into</param> public void CopyTo(Buffer destination, int dstOffset) { + CopyFromDependantVirtualBuffers(); _context.Renderer.Pipeline.CopyBuffer(Handle, destination.Handle, 0, dstOffset, (int)Size); } @@ -536,7 +548,7 @@ namespace Ryujinx.Graphics.Gpu.Memory using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size); // TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers. - _physicalMemory.WriteUntracked(address, data.Get()); + _physicalMemory.WriteUntracked(address, CopyFromDependantVirtualBuffers(data.Get(), address, size)); } /// <summary> @@ -617,6 +629,207 @@ namespace Ryujinx.Graphics.Gpu.Memory UnmappedSequence++; } + /// <summary> + /// Adds a virtual buffer dependency, indicating that a virtual buffer depends on data from this buffer. + /// </summary> + /// <param name="virtualBuffer">Dependant virtual buffer</param> + public void AddVirtualDependency(MultiRangeBuffer virtualBuffer) + { + _virtualDependenciesLock.EnterWriteLock(); + + try + { + (_virtualDependencies ??= new()).Add(virtualBuffer); + } + finally + { + _virtualDependenciesLock.ExitWriteLock(); + } + } + + /// <summary> + /// Removes a virtual buffer dependency, indicating that a virtual buffer no longer depends on data from this buffer. + /// </summary> + /// <param name="virtualBuffer">Dependant virtual buffer</param> + public void RemoveVirtualDependency(MultiRangeBuffer virtualBuffer) + { + _virtualDependenciesLock.EnterWriteLock(); + + try + { + if (_virtualDependencies != null) + { + _virtualDependencies.Remove(virtualBuffer); + + if (_virtualDependencies.Count == 0) + { + _virtualDependencies = null; + } + } + } + finally + { + _virtualDependenciesLock.ExitWriteLock(); + } + } + + /// <summary> + /// Copies the buffer data to all virtual buffers that depends on it. + /// </summary> + public void CopyToDependantVirtualBuffers() + { + CopyToDependantVirtualBuffers(Address, Size); + } + + /// <summary> + /// Copies the buffer data inside the specifide range to all virtual buffers that depends on it. + /// </summary> + /// <param name="address">Address of the range</param> + /// <param name="size">Size of the range in bytes</param> + public void CopyToDependantVirtualBuffers(ulong address, ulong size) + { + if (_virtualDependencies != null) + { + foreach (var virtualBuffer in _virtualDependencies) + { + CopyToDependantVirtualBuffer(virtualBuffer, address, size); + } + } + } + + /// <summary> + /// Copies all modified ranges from all virtual buffers back into this buffer. + /// </summary> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void CopyFromDependantVirtualBuffers() + { + if (_virtualDependencies != null) + { + CopyFromDependantVirtualBuffersImpl(); + } + } + + /// <summary> + /// Copies all modified ranges from all virtual buffers back into this buffer. + /// </summary> + [MethodImpl(MethodImplOptions.NoInlining)] + private void CopyFromDependantVirtualBuffersImpl() + { + foreach (var virtualBuffer in _virtualDependencies.OrderBy(x => x.ModificationSequenceNumber)) + { + virtualBuffer.ConsumeModifiedRegion(this, (mAddress, mSize) => + { + // Get offset inside both this and the virtual buffer. + // Note that sometimes there is no right answer for the virtual offset, + // as the same physical range might be mapped multiple times inside a virtual buffer. + // We just assume it does not happen in practice as it can only be implemented correctly + // when the host has support for proper sparse mapping. + + ulong mEndAddress = mAddress + mSize; + mAddress = Math.Max(mAddress, Address); + mSize = Math.Min(mEndAddress, EndAddress) - mAddress; + + int physicalOffset = (int)(mAddress - Address); + int virtualOffset = virtualBuffer.Range.FindOffset(new(mAddress, mSize)); + + _context.Renderer.Pipeline.CopyBuffer(virtualBuffer.Handle, Handle, virtualOffset, physicalOffset, (int)mSize); + }); + } + } + + /// <summary> + /// Copies all overlapping modified ranges from all virtual buffers back into this buffer, and returns an updated span with the data. + /// </summary> + /// <param name="dataSpan">Span where the unmodified data will be taken from for the output</param> + /// <param name="address">Address of the region to copy</param> + /// <param name="size">Size of the region to copy in bytes</param> + /// <returns>A span with <paramref name="dataSpan"/>, and the data for all modified ranges if any</returns> + private ReadOnlySpan<byte> CopyFromDependantVirtualBuffers(ReadOnlySpan<byte> dataSpan, ulong address, ulong size) + { + _virtualDependenciesLock.EnterReadLock(); + + try + { + if (_virtualDependencies != null) + { + byte[] storage = dataSpan.ToArray(); + + foreach (var virtualBuffer in _virtualDependencies.OrderBy(x => x.ModificationSequenceNumber)) + { + virtualBuffer.ConsumeModifiedRegion(address, size, (mAddress, mSize) => + { + // Get offset inside both this and the virtual buffer. + // Note that sometimes there is no right answer for the virtual offset, + // as the same physical range might be mapped multiple times inside a virtual buffer. + // We just assume it does not happen in practice as it can only be implemented correctly + // when the host has support for proper sparse mapping. + + ulong mEndAddress = mAddress + mSize; + mAddress = Math.Max(mAddress, address); + mSize = Math.Min(mEndAddress, address + size) - mAddress; + + int physicalOffset = (int)(mAddress - Address); + int virtualOffset = virtualBuffer.Range.FindOffset(new(mAddress, mSize)); + + _context.Renderer.Pipeline.CopyBuffer(virtualBuffer.Handle, Handle, virtualOffset, physicalOffset, (int)size); + virtualBuffer.GetData(storage.AsSpan().Slice((int)(mAddress - address), (int)mSize), virtualOffset, (int)mSize); + }); + } + + dataSpan = storage; + } + } + finally + { + _virtualDependenciesLock.ExitReadLock(); + } + + return dataSpan; + } + + /// <summary> + /// Copies the buffer data to the specified virtual buffer. + /// </summary> + /// <param name="virtualBuffer">Virtual buffer to copy the data into</param> + public void CopyToDependantVirtualBuffer(MultiRangeBuffer virtualBuffer) + { + CopyToDependantVirtualBuffer(virtualBuffer, Address, Size); + } + + /// <summary> + /// Copies the buffer data inside the given range to the specified virtual buffer. + /// </summary> + /// <param name="virtualBuffer">Virtual buffer to copy the data into</param> + /// <param name="address">Address of the range</param> + /// <param name="size">Size of the range in bytes</param> + public void CopyToDependantVirtualBuffer(MultiRangeBuffer virtualBuffer, ulong address, ulong size) + { + // Broadcast data to all ranges of the virtual buffer that are contained inside this buffer. + + ulong lastOffset = 0; + + while (virtualBuffer.TryGetPhysicalOffset(this, lastOffset, out ulong srcOffset, out ulong dstOffset, out ulong copySize)) + { + ulong innerOffset = address - Address; + ulong innerEndOffset = (address + size) - Address; + + lastOffset = dstOffset + copySize; + + // Clamp range to the specified range. + ulong copySrcOffset = Math.Max(srcOffset, innerOffset); + ulong copySrcEndOffset = Math.Min(innerEndOffset, srcOffset + copySize); + + if (copySrcEndOffset > copySrcOffset) + { + copySize = copySrcEndOffset - copySrcOffset; + dstOffset += copySrcOffset - srcOffset; + srcOffset = copySrcOffset; + + _context.Renderer.Pipeline.CopyBuffer(Handle, virtualBuffer.Handle, (int)srcOffset, (int)dstOffset, (int)copySize); + } + } + } + /// <summary> /// Increments the buffer reference count. /// </summary> diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index bd9aa39c8..c6284780d 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -3,6 +3,7 @@ using Ryujinx.Memory.Range; using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Gpu.Memory { @@ -46,6 +47,7 @@ namespace Ryujinx.Graphics.Gpu.Memory private readonly Dictionary<ulong, BufferCacheEntry> _dirtyCache; private readonly Dictionary<ulong, BufferCacheEntry> _modifiedCache; private bool _pruneCaches; + private int _virtualModifiedSequenceNumber; public event Action NotifyBuffersModified; @@ -125,7 +127,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// <summary> /// Performs address translation of the GPU virtual address, and creates - /// new buffers, if needed, for the specified range. + /// new physical and virtual buffers, if needed, for the specified range. /// </summary> /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param> /// <param name="gpuVa">Start GPU virtual address of the buffer</param> @@ -138,12 +140,10 @@ namespace Ryujinx.Graphics.Gpu.Memory return new MultiRange(MemoryManager.PteUnmapped, size); } - bool supportsSparse = _context.Capabilities.SupportsSparseBuffer; - // Fast path not taken for non-contiguous ranges, // since multi-range buffers are not coalesced, so a buffer that covers // the entire cached range might not actually exist. - if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) && + if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) && range.Count == 1) { return range; @@ -154,6 +154,50 @@ namespace Ryujinx.Graphics.Gpu.Memory return range; } + /// <summary> + /// Performs address translation of the GPU virtual address, and creates + /// new physical buffers, if needed, for the specified range. + /// </summary> + /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param> + /// <param name="gpuVa">Start GPU virtual address of the buffer</param> + /// <param name="size">Size in bytes of the buffer</param> + /// <returns>Physical ranges of the buffer, after address translation</returns> + public MultiRange TranslateAndCreateMultiBuffersPhysicalOnly(MemoryManager memoryManager, ulong gpuVa, ulong size) + { + if (gpuVa == 0) + { + return new MultiRange(MemoryManager.PteUnmapped, size); + } + + // Fast path not taken for non-contiguous ranges, + // since multi-range buffers are not coalesced, so a buffer that covers + // the entire cached range might not actually exist. + if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) && + range.Count == 1) + { + return range; + } + + for (int i = 0; i < range.Count; i++) + { + MemoryRange subRange = range.GetSubRange(i); + + if (subRange.Address != MemoryManager.PteUnmapped) + { + if (range.Count > 1) + { + CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize); + } + else + { + CreateBuffer(subRange.Address, subRange.Size); + } + } + } + + return range; + } + /// <summary> /// Creates a new buffer for the specified range, if it does not yet exist. /// This can be used to ensure the existance of a buffer. @@ -263,41 +307,108 @@ namespace Ryujinx.Graphics.Gpu.Memory } } - BufferRange[] storages = new BufferRange[range.Count]; + MultiRangeBuffer multiRangeBuffer; + MemoryRange[] alignedSubRanges = new MemoryRange[range.Count]; ulong alignmentMask = SparseBufferAlignmentSize - 1; - for (int i = 0; i < range.Count; i++) + if (_context.Capabilities.SupportsSparseBuffer) { - MemoryRange subRange = range.GetSubRange(i); + BufferRange[] storages = new BufferRange[range.Count]; + + for (int i = 0; i < range.Count; i++) + { + MemoryRange subRange = range.GetSubRange(i); + + if (subRange.Address != MemoryManager.PteUnmapped) + { + ulong endAddress = subRange.Address + subRange.Size; + + ulong alignedAddress = subRange.Address & ~alignmentMask; + ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; + ulong alignedSize = alignedEndAddress - alignedAddress; + + Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); + BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); + + alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); + storages[i] = bufferRange; + } + else + { + ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask; + + alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize); + storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize); + } + } + + multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages); + } + else + { + for (int i = 0; i < range.Count; i++) + { + MemoryRange subRange = range.GetSubRange(i); + + if (subRange.Address != MemoryManager.PteUnmapped) + { + ulong endAddress = subRange.Address + subRange.Size; + + ulong alignedAddress = subRange.Address & ~alignmentMask; + ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; + ulong alignedSize = alignedEndAddress - alignedAddress; + + alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); + } + else + { + ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask; + + alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize); + } + } + + multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges)); + + UpdateVirtualBufferDependencies(multiRangeBuffer); + } + + _multiRangeBuffers.Add(multiRangeBuffer); + } + + /// <summary> + /// Adds two-way dependencies to all physical buffers contained within a given virtual buffer. + /// </summary> + /// <param name="virtualBuffer">Virtual buffer to have dependencies added</param> + private void UpdateVirtualBufferDependencies(MultiRangeBuffer virtualBuffer) + { + virtualBuffer.ClearPhysicalDependencies(); + + ulong dstOffset = 0; + + HashSet<Buffer> physicalBuffers = new(); + + for (int i = 0; i < virtualBuffer.Range.Count; i++) + { + MemoryRange subRange = virtualBuffer.Range.GetSubRange(i); if (subRange.Address != MemoryManager.PteUnmapped) { - ulong endAddress = subRange.Address + subRange.Size; + Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); - ulong alignedAddress = subRange.Address & ~alignmentMask; - ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; - ulong alignedSize = alignedEndAddress - alignedAddress; - - Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); - BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); - - storages[i] = bufferRange; - alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); + virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size); + physicalBuffers.Add(buffer); } - else - { - ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask; - storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize); - alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize); - } + dstOffset += subRange.Size; } - MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages); - - _multiRangeBuffers.Add(multiRangeBuffer); + foreach (var buffer in physicalBuffers) + { + buffer.CopyToDependantVirtualBuffer(virtualBuffer); + } } /// <summary> @@ -620,8 +731,8 @@ namespace Ryujinx.Graphics.Gpu.Memory /// <param name="size">Size in bytes of the copy</param> public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size) { - MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size); - MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size); + MultiRange srcRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, srcVa, size); + MultiRange dstRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, dstVa, size); if (srcRange.Count == 1 && dstRange.Count == 1) { @@ -701,6 +812,8 @@ namespace Ryujinx.Graphics.Gpu.Memory dstBuffer.ClearModified(dstAddress, size); memoryManager.Physical.WriteTrackedResource(dstAddress, memoryManager.Physical.GetSpan(srcAddress, (int)size), ResourceKind.Buffer); } + + dstBuffer.CopyToDependantVirtualBuffers(dstAddress, size); } /// <summary> @@ -715,7 +828,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// <param name="value">Value to be written into the buffer</param> public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value) { - MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size); + MultiRange range = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, gpuVa, size); for (int index = 0; index < range.Count; index++) { @@ -727,6 +840,8 @@ namespace Ryujinx.Graphics.Gpu.Memory _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value); memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer); + + buffer.CopyToDependantVirtualBuffers(subRange.Address, subRange.Size); } } @@ -806,6 +921,11 @@ namespace Ryujinx.Graphics.Gpu.Memory } } + if (write && buffer != null && !_context.Capabilities.SupportsSparseBuffer) + { + buffer.AddModifiedRegion(range, ++_virtualModifiedSequenceNumber); + } + return buffer; } @@ -825,6 +945,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { buffer = _buffers.FindFirstOverlap(address, size); + buffer.CopyFromDependantVirtualBuffers(); buffer.SynchronizeMemory(address, size); if (write) @@ -849,14 +970,14 @@ namespace Ryujinx.Graphics.Gpu.Memory if (range.Count == 1) { MemoryRange subRange = range.GetSubRange(0); - SynchronizeBufferRange(subRange.Address, subRange.Size); + SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: true); } else { for (int index = 0; index < range.Count; index++) { MemoryRange subRange = range.GetSubRange(index); - SynchronizeBufferRange(subRange.Address, subRange.Size); + SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: false); } } } @@ -866,12 +987,19 @@ namespace Ryujinx.Graphics.Gpu.Memory /// </summary> /// <param name="address">Start address of the memory range</param> /// <param name="size">Size in bytes of the memory range</param> - private void SynchronizeBufferRange(ulong address, ulong size) + /// <param name="copyBackVirtual">Whether virtual buffers that uses this buffer as backing memory should have its data copied back if modified</param> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SynchronizeBufferRange(ulong address, ulong size, bool copyBackVirtual) { if (size != 0) { Buffer buffer = _buffers.FindFirstOverlap(address, size); + if (copyBackVirtual) + { + buffer.CopyFromDependantVirtualBuffers(); + } + buffer.SynchronizeMemory(address, size); } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs index 74d527051..30f87813f 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs @@ -40,9 +40,9 @@ namespace Ryujinx.Graphics.Gpu.Memory internal PhysicalMemory Physical { get; } /// <summary> - /// Virtual buffer cache. + /// Virtual range cache. /// </summary> - internal VirtualBufferCache VirtualBufferCache { get; } + internal VirtualRangeCache VirtualRangeCache { get; } /// <summary> /// Cache of GPU counters. @@ -56,12 +56,12 @@ namespace Ryujinx.Graphics.Gpu.Memory internal MemoryManager(PhysicalMemory physicalMemory) { Physical = physicalMemory; - VirtualBufferCache = new VirtualBufferCache(this); + VirtualRangeCache = new VirtualRangeCache(this); CounterCache = new CounterCache(); _pageTable = new ulong[PtLvl0Size][]; MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler; MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler; - MemoryUnmapped += VirtualBufferCache.MemoryUnmappedHandler; + MemoryUnmapped += VirtualRangeCache.MemoryUnmappedHandler; MemoryUnmapped += CounterCache.MemoryUnmappedHandler; } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs index e039a7a43..d92b0836e 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs @@ -1,6 +1,7 @@ using Ryujinx.Graphics.GAL; using Ryujinx.Memory.Range; using System; +using System.Collections.Generic; namespace Ryujinx.Graphics.Gpu.Memory { @@ -21,12 +22,73 @@ namespace Ryujinx.Graphics.Gpu.Memory /// </summary> public MultiRange Range { get; } + /// <summary> + /// Ever increasing counter value indicating when the buffer was modified relative to other buffers. + /// </summary> + public int ModificationSequenceNumber { get; private set; } + + /// <summary> + /// Physical buffer dependency entry. + /// </summary> + private readonly struct PhysicalDependency + { + /// <summary> + /// Physical buffer. + /// </summary> + public readonly Buffer PhysicalBuffer; + + /// <summary> + /// Offset of the range on the physical buffer. + /// </summary> + public readonly ulong PhysicalOffset; + + /// <summary> + /// Offset of the range on the virtual buffer. + /// </summary> + public readonly ulong VirtualOffset; + + /// <summary> + /// Size of the range. + /// </summary> + public readonly ulong Size; + + /// <summary> + /// Creates a new physical dependency. + /// </summary> + /// <param name="physicalBuffer">Physical buffer</param> + /// <param name="physicalOffset">Offset of the range on the physical buffer</param> + /// <param name="virtualOffset">Offset of the range on the virtual buffer</param> + /// <param name="size">Size of the range</param> + public PhysicalDependency(Buffer physicalBuffer, ulong physicalOffset, ulong virtualOffset, ulong size) + { + PhysicalBuffer = physicalBuffer; + PhysicalOffset = physicalOffset; + VirtualOffset = virtualOffset; + Size = size; + } + } + + private List<PhysicalDependency> _dependencies; + private BufferModifiedRangeList _modifiedRanges = null; + /// <summary> /// Creates a new instance of the buffer. /// </summary> /// <param name="context">GPU context that the buffer belongs to</param> /// <param name="range">Range of memory where the data is mapped</param> - /// <param name="storages">Backing memory for the buffers</param> + public MultiRangeBuffer(GpuContext context, MultiRange range) + { + _context = context; + Range = range; + Handle = context.Renderer.CreateBuffer((int)range.GetSize()); + } + + /// <summary> + /// Creates a new instance of the buffer. + /// </summary> + /// <param name="context">GPU context that the buffer belongs to</param> + /// <param name="range">Range of memory where the data is mapped</param> + /// <param name="storages">Backing memory for the buffer</param> public MultiRangeBuffer(GpuContext context, MultiRange range, ReadOnlySpan<BufferRange> storages) { _context = context; @@ -49,11 +111,134 @@ namespace Ryujinx.Graphics.Gpu.Memory return new BufferRange(Handle, offset, (int)range.GetSize()); } + /// <summary> + /// Removes all physical buffer dependencies. + /// </summary> + public void ClearPhysicalDependencies() + { + _dependencies?.Clear(); + } + + /// <summary> + /// Adds a physical buffer dependency. + /// </summary> + /// <param name="buffer">Physical buffer to be added</param> + /// <param name="rangeAddress">Address inside the physical buffer where the virtual buffer range is located</param> + /// <param name="dstOffset">Offset inside the virtual buffer where the physical range is located</param> + /// <param name="rangeSize">Size of the range in bytes</param> + public void AddPhysicalDependency(Buffer buffer, ulong rangeAddress, ulong dstOffset, ulong rangeSize) + { + (_dependencies ??= new()).Add(new(buffer, rangeAddress - buffer.Address, dstOffset, rangeSize)); + buffer.AddVirtualDependency(this); + } + + /// <summary> + /// Tries to get the physical range corresponding to the given physical buffer. + /// </summary> + /// <param name="buffer">Physical buffer</param> + /// <param name="minimumVirtOffset">Minimum virtual offset that a range match can have</param> + /// <param name="physicalOffset">Physical offset of the match</param> + /// <param name="virtualOffset">Virtual offset of the match, always greater than or equal <paramref name="minimumVirtOffset"/></param> + /// <param name="size">Size of the range match</param> + /// <returns>True if a match was found for the given parameters, false otherwise</returns> + public bool TryGetPhysicalOffset(Buffer buffer, ulong minimumVirtOffset, out ulong physicalOffset, out ulong virtualOffset, out ulong size) + { + physicalOffset = 0; + virtualOffset = 0; + size = 0; + + if (_dependencies != null) + { + foreach (var dependency in _dependencies) + { + if (dependency.PhysicalBuffer == buffer && dependency.VirtualOffset >= minimumVirtOffset) + { + physicalOffset = dependency.PhysicalOffset; + virtualOffset = dependency.VirtualOffset; + size = dependency.Size; + + return true; + } + } + } + + return false; + } + + /// <summary> + /// Adds a modified virtual memory range. + /// </summary> + /// <remarks> + /// This is only required when the host does not support sparse buffers, otherwise only physical buffers need to track modification. + /// </remarks> + /// <param name="range">Modified range</param> + /// <param name="modifiedSequenceNumber">ModificationSequenceNumber</param> + public void AddModifiedRegion(MultiRange range, int modifiedSequenceNumber) + { + _modifiedRanges ??= new(_context, null, null); + + for (int i = 0; i < range.Count; i++) + { + MemoryRange subRange = range.GetSubRange(i); + + _modifiedRanges.SignalModified(subRange.Address, subRange.Size); + } + + ModificationSequenceNumber = modifiedSequenceNumber; + } + + /// <summary> + /// Calls the specified <paramref name="rangeAction"/> for all modified ranges that overlaps with <paramref name="buffer"/>. + /// </summary> + /// <param name="buffer">Buffer to have its range checked</param> + /// <param name="rangeAction">Action to perform for modified ranges</param> + public void ConsumeModifiedRegion(Buffer buffer, Action<ulong, ulong> rangeAction) + { + ConsumeModifiedRegion(buffer.Address, buffer.Size, rangeAction); + } + + /// <summary> + /// Calls the specified <paramref name="rangeAction"/> for all modified ranges that overlaps with <paramref name="address"/> and <paramref name="size"/>. + /// </summary> + /// <param name="address">Address of the region to consume</param> + /// <param name="size">Size of the region to consume</param> + /// <param name="rangeAction">Action to perform for modified ranges</param> + public void ConsumeModifiedRegion(ulong address, ulong size, Action<ulong, ulong> rangeAction) + { + if (_modifiedRanges != null) + { + _modifiedRanges.GetRanges(address, size, rangeAction); + _modifiedRanges.Clear(address, size); + } + } + + /// <summary> + /// Gets data from the specified region of the buffer, and places it on <paramref name="output"/>. + /// </summary> + /// <param name="output">Span to put the data into</param> + /// <param name="offset">Offset of the buffer to get the data from</param> + /// <param name="size">Size of the data in bytes</param> + public void GetData(Span<byte> output, int offset, int size) + { + using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, size); + data.Get().CopyTo(output); + } + /// <summary> /// Disposes the host buffer. /// </summary> public void Dispose() { + if (_dependencies != null) + { + foreach (var dependency in _dependencies) + { + dependency.PhysicalBuffer.RemoveVirtualDependency(this); + } + + _dependencies = null; + } + _context.Renderer.DeleteBuffer(Handle); } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs similarity index 92% rename from src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs rename to src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs index 858c5e3b0..889f5c9ca 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs @@ -6,9 +6,9 @@ using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory { /// <summary> - /// Virtual buffer cache. + /// Virtual range cache. /// </summary> - class VirtualBufferCache + class VirtualRangeCache { private readonly MemoryManager _memoryManager; @@ -68,10 +68,10 @@ namespace Ryujinx.Graphics.Gpu.Memory private int _hasDeferredUnmaps; /// <summary> - /// Creates a new instance of the virtual buffer cache. + /// Creates a new instance of the virtual range cache. /// </summary> - /// <param name="memoryManager">Memory manager that the virtual buffer cache belongs to</param> - public VirtualBufferCache(MemoryManager memoryManager) + /// <param name="memoryManager">Memory manager that the virtual range cache belongs to</param> + public VirtualRangeCache(MemoryManager memoryManager) { _memoryManager = memoryManager; _virtualRanges = new RangeList<VirtualRange>(); @@ -102,10 +102,9 @@ namespace Ryujinx.Graphics.Gpu.Memory /// </summary> /// <param name="gpuVa">GPU virtual address to get the physical range from</param> /// <param name="size">Size in bytes of the region</param> - /// <param name="supportsSparse">Indicates host support for sparse buffer mapping of non-contiguous ranges</param> /// <param name="range">Physical range for the specified GPU virtual region</param> /// <returns>True if the range already existed, false if a new one was created and added</returns> - public bool TryGetOrAddRange(ulong gpuVa, ulong size, bool supportsSparse, out MultiRange range) + public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range) { VirtualRange[] overlaps = _virtualRangeOverlaps; int overlapsCount; @@ -158,7 +157,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } else { - found = true; + found = overlap0.Range.Count == 1 || IsSparseAligned(overlap0.Range); range = overlap0.Range.Slice(gpuVa - overlap0.Address, size); } } @@ -175,11 +174,10 @@ namespace Ryujinx.Graphics.Gpu.Memory ShrinkOverlapsBufferIfNeeded(); // If the the range is not properly aligned for sparse mapping, - // or if the host does not support sparse mapping, let's just - // force it to a single range. + // let's just force it to a single range. // This might cause issues in some applications that uses sparse // mappings. - if (!IsSparseAligned(range) || !supportsSparse) + if (!IsSparseAligned(range)) { range = new MultiRange(range.GetSubRange(0).Address, size); }