emu-mirrors/ryujinx-final
Archived
0
0
Fork 0
Code Activity

Vulkan: Buffer Mirrors for MacOS performance (#4899)

Browse source 
* Initial implementation of buffer mirrors

Generally slower right now, goal is to reduce render passes in games that do inline updates

Fix support buffer mirrors

Reintroduce vertex buffer mirror

Add storage buffer support

Optimisation part 1

More optimisation

Avoid useless data copies.

Remove unused cbIndex stuff

Properly set write flag for storage buffers.

Fix minor issues

Not sure why this was here.

Fix BufferRangeList

Fix some big issues

Align storage buffers rather than getting full buffer as a range

Improves mirrorability of read-only storage buffers

Increase staging buffer size, as it now contains mirrors

Fix some issues with buffers not updating

Fix buffer SetDataUnchecked offset for one of the paths when using mirrors

Fix buffer mirrors interaction with buffer textures

Fix mirror rebinding

Move GetBuffer calls on indirect draws before BeginRenderPass to avoid draws without render pass

Fix mirrors rebase

Fix rebase 2023

* Fix crash when using stale vertex buffer

Similar to `Get` with a size that's too large, just treat it as a clamp.

* Explicitly set support buffer as mirrorable

* Address feedback

* Remove unused fragment of MVK workaround

* Replace logging for staging buffer OOM

* Address format issues

* Address more format issues

* Mini cleanup

* Address more things

* Rename BufferRangeList

* Support bounding range for ClearMirrors and UploadPendingData

* Add maximum size for vertex buffer mirrors

* Enable index buffer mirrors

Enabled on all platforms for the IbStreamer.

* Feedback

* Remove mystery BufferCache change

Probably macos related?

* Fix mirrors not creating when staging buffer is empty.

* Change log level to debug
This commit is contained in:
riperiperi 2023-08-14 18:18:47 +01:00 committed by GitHub
parent 550fd4a733
commit 492a046335
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 1285 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Ryujinx.Graphics.GAL/BufferAccess.cs
View file

@ -4,5 +4,6 @@ namespace Ryujinx.Graphics.GAL
{ {
Default, Default,
FlushPersistent, FlushPersistent,
Stream
} }
} }

4
src/Ryujinx.Graphics.GAL/BufferRange.cs
View file

@ -10,12 +10,14 @@ namespace Ryujinx.Graphics.GAL
public int Offset { get; } public int Offset { get; }
public int Size { get; } public int Size { get; }
public bool Write { get; }
public BufferRange(BufferHandle handle, int offset, int size) public BufferRange(BufferHandle handle, int offset, int size, bool write = false)
{ {
Handle = handle; Handle = handle;
Offset = offset; Offset = offset;
Size = size; Size = size;
Write = write;
} }
} }
} }

8
src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs
View file

@ -113,7 +113,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
internal BufferRange MapBufferRange(BufferRange range) internal BufferRange MapBufferRange(BufferRange range)
{ {
return new BufferRange(MapBuffer(range.Handle), range.Offset, range.Size); return new BufferRange(MapBuffer(range.Handle), range.Offset, range.Size, range.Write);
} }
internal Span<BufferRange> MapBufferRanges(Span<BufferRange> ranges) internal Span<BufferRange> MapBufferRanges(Span<BufferRange> ranges)
@ -131,7 +131,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
result = BufferHandle.Null; result = BufferHandle.Null;
} }
range = new BufferRange(result, range.Offset, range.Size); range = new BufferRange(result, range.Offset, range.Size, range.Write);
} }
} }
@ -154,7 +154,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
result = BufferHandle.Null; result = BufferHandle.Null;
} }
assignment = new BufferAssignment(ranges[i].Binding, new BufferRange(result, range.Offset, range.Size)); assignment = new BufferAssignment(ranges[i].Binding, new BufferRange(result, range.Offset, range.Size, range.Write));
} }
} }
@ -176,7 +176,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
result = BufferHandle.Null; result = BufferHandle.Null;
} }
range = new BufferRange(result, range.Offset, range.Size); range = new BufferRange(result, range.Offset, range.Size, range.Write);
ranges[i] = new VertexBufferDescriptor(range, ranges[i].Stride, ranges[i].Divisor); ranges[i] = new VertexBufferDescriptor(range, ranges[i].Stride, ranges[i].Divisor);
} }

4
src/Ryujinx.Graphics.Gpu/Engine/Threed/IbStreamer.cs
View file

@ -171,7 +171,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
if (_inlineIndexBuffer == BufferHandle.Null) if (_inlineIndexBuffer == BufferHandle.Null)
{ {
_inlineIndexBuffer = renderer.CreateBuffer(size); _inlineIndexBuffer = renderer.CreateBuffer(size, BufferAccess.Stream);
_inlineIndexBufferSize = size; _inlineIndexBufferSize = size;
} }
else if (_inlineIndexBufferSize < size) else if (_inlineIndexBufferSize < size)
@ -179,7 +179,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
BufferHandle oldBuffer = _inlineIndexBuffer; BufferHandle oldBuffer = _inlineIndexBuffer;
int oldSize = _inlineIndexBufferSize; int oldSize = _inlineIndexBufferSize;
_inlineIndexBuffer = renderer.CreateBuffer(size); _inlineIndexBuffer = renderer.CreateBuffer(size, BufferAccess.Stream);
_inlineIndexBufferSize = size; _inlineIndexBufferSize = size;
renderer.Pipeline.CopyBuffer(oldBuffer, _inlineIndexBuffer, 0, 0, oldSize); renderer.Pipeline.CopyBuffer(oldBuffer, _inlineIndexBuffer, 0, 0, oldSize);

14
src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
View file

@ -140,18 +140,21 @@ namespace Ryujinx.Graphics.Gpu.Memory
} }
/// <summary> /// <summary>
/// Gets a sub-range from the buffer, from a start address till the end of the buffer. /// Gets a sub-range from the buffer, from a start address til a page boundary after the given size.
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// This can be used to bind and use sub-ranges of the buffer on the host API. /// This can be used to bind and use sub-ranges of the buffer on the host API.
/// </remarks> /// </remarks>
/// <param name="address">Start address of the sub-range, must be greater than or equal to the buffer address</param> /// <param name="address">Start address of the sub-range, must be greater than or equal to the buffer address</param>
/// <param name="size">Size in bytes of the sub-range, must be less than or equal to the buffer size</param>
/// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer sub-range</returns> /// <returns>The buffer sub-range</returns>
public BufferRange GetRange(ulong address) public BufferRange GetRangeAligned(ulong address, ulong size, bool write)
{ {
ulong end = ((address + size + MemoryManager.PageMask) & ~MemoryManager.PageMask) - Address;
ulong offset = address - Address; ulong offset = address - Address;
return new BufferRange(Handle, (int)offset, (int)(Size - offset)); return new BufferRange(Handle, (int)offset, (int)(end - offset), write);
} }
/// <summary> /// <summary>
@ -162,12 +165,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </remarks> /// </remarks>
/// <param name="address">Start address of the sub-range, must be greater than or equal to the buffer address</param> /// <param name="address">Start address of the sub-range, must be greater than or equal to the buffer address</param>
/// <param name="size">Size in bytes of the sub-range, must be less than or equal to the buffer size</param> /// <param name="size">Size in bytes of the sub-range, must be less than or equal to the buffer size</param>
/// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer sub-range</returns> /// <returns>The buffer sub-range</returns>
public BufferRange GetRange(ulong address, ulong size) public BufferRange GetRange(ulong address, ulong size, bool write)
{ {
int offset = (int)(address - Address); int offset = (int)(address - Address);
return new BufferRange(Handle, offset, (int)size); return new BufferRange(Handle, offset, (int)size, write);
} }
/// <summary> /// <summary>

8
src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
View file

@ -372,15 +372,15 @@ namespace Ryujinx.Graphics.Gpu.Memory
} }
/// <summary> /// <summary>
/// Gets a buffer sub-range starting at a given memory address. /// Gets a buffer sub-range from a start address til a page boundary after the given size.
/// </summary> /// </summary>
/// <param name="address">Start address of the memory range</param> /// <param name="address">Start address of the memory range</param>
/// <param name="size">Size in bytes of the memory range</param> /// <param name="size">Size in bytes of the memory range</param>
/// <param name="write">Whether the buffer will be written to by this use</param> /// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer sub-range starting at the given memory address</returns> /// <returns>The buffer sub-range starting at the given memory address</returns>
public BufferRange GetBufferRangeTillEnd(ulong address, ulong size, bool write = false) public BufferRange GetBufferRangeAligned(ulong address, ulong size, bool write = false)
{ {
return GetBuffer(address, size, write).GetRange(address); return GetBuffer(address, size, write).GetRangeAligned(address, size, write);
} }
/// <summary> /// <summary>
@ -392,7 +392,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>The buffer sub-range for the given range</returns> /// <returns>The buffer sub-range for the given range</returns>
public BufferRange GetBufferRange(ulong address, ulong size, bool write = false) public BufferRange GetBufferRange(ulong address, ulong size, bool write = false)
{ {
return GetBuffer(address, size, write).GetRange(address, size); return GetBuffer(address, size, write).GetRange(address, size, write);
} }
/// <summary> /// <summary>

6
src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs
View file

@ -614,7 +614,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (_tfInfoBuffer == BufferHandle.Null) if (_tfInfoBuffer == BufferHandle.Null)
{ {
_tfInfoBuffer = _context.Renderer.CreateBuffer(TfInfoBufferSize); _tfInfoBuffer = _context.Renderer.CreateBuffer(TfInfoBufferSize, BufferAccess.Stream);
} }
buffers[0] = new BufferAssignment(0, new BufferRange(_tfInfoBuffer, 0, TfInfoBufferSize)); buffers[0] = new BufferAssignment(0, new BufferRange(_tfInfoBuffer, 0, TfInfoBufferSize));
@ -727,7 +727,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
var range = isStorage var range = isStorage
? bufferCache.GetBufferRangeTillEnd(bounds.Address, bounds.Size, isWrite) ? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite)
: bufferCache.GetBufferRange(bounds.Address, bounds.Size); : bufferCache.GetBufferRange(bounds.Address, bounds.Size);
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range); ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
@ -764,7 +764,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
var range = isStorage var range = isStorage
? bufferCache.GetBufferRangeTillEnd(bounds.Address, bounds.Size, isWrite) ? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite)
: bufferCache.GetBufferRange(bounds.Address, bounds.Size); : bufferCache.GetBufferRange(bounds.Address, bounds.Size);
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range); ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);

2
src/Ryujinx.Graphics.Gpu/Memory/SupportBufferUpdater.cs
View file

@ -228,7 +228,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
if (_handle == BufferHandle.Null) if (_handle == BufferHandle.Null)
{ {
_handle = _renderer.CreateBuffer(SupportBuffer.RequiredSize); _handle = _renderer.CreateBuffer(SupportBuffer.RequiredSize, BufferAccess.Stream);
_renderer.Pipeline.ClearBuffer(_handle, 0, SupportBuffer.RequiredSize, 0); _renderer.Pipeline.ClearBuffer(_handle, 0, SupportBuffer.RequiredSize, 0);
var range = new BufferRange(_handle, 0, SupportBuffer.RequiredSize); var range = new BufferRange(_handle, 0, SupportBuffer.RequiredSize);

24
src/Ryujinx.Graphics.Vulkan/Auto.cs
View file

@ -18,6 +18,12 @@ namespace Ryujinx.Graphics.Vulkan
void AddCommandBufferDependencies(CommandBufferScoped cbs); void AddCommandBufferDependencies(CommandBufferScoped cbs);
} }
interface IMirrorable<T> where T : IDisposable
{
Auto<T> GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored);
void ClearMirrors(CommandBufferScoped cbs, int offset, int size);
}
class Auto<T> : IAutoPrivate, IDisposable where T : IDisposable class Auto<T> : IAutoPrivate, IDisposable where T : IDisposable
{ {
private int _referenceCount; private int _referenceCount;
@ -26,6 +32,7 @@ namespace Ryujinx.Graphics.Vulkan
private readonly BitMap _cbOwnership; private readonly BitMap _cbOwnership;
private readonly MultiFenceHolder _waitable; private readonly MultiFenceHolder _waitable;
private readonly IAutoPrivate[] _referencedObjs; private readonly IAutoPrivate[] _referencedObjs;
private readonly IMirrorable<T> _mirrorable;
private bool _disposed; private bool _disposed;
private bool _destroyed; private bool _destroyed;
@ -37,6 +44,11 @@ namespace Ryujinx.Graphics.Vulkan
_cbOwnership = new BitMap(CommandBufferPool.MaxCommandBuffers); _cbOwnership = new BitMap(CommandBufferPool.MaxCommandBuffers);
} }
public Auto(T value, IMirrorable<T> mirrorable, MultiFenceHolder waitable, params IAutoPrivate[] referencedObjs) : this(value, waitable, referencedObjs)
{
_mirrorable = mirrorable;
}
public Auto(T value, MultiFenceHolder waitable, params IAutoPrivate[] referencedObjs) : this(value) public Auto(T value, MultiFenceHolder waitable, params IAutoPrivate[] referencedObjs) : this(value)
{ {
_waitable = waitable; _waitable = waitable;
@ -48,9 +60,17 @@ namespace Ryujinx.Graphics.Vulkan
} }
} }
public T Get(CommandBufferScoped cbs, int offset, int size) public T GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored)
{ {
_waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size); var mirror = _mirrorable.GetMirrorable(cbs, ref offset, size, out mirrored);
mirror._waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size, false);
return mirror.Get(cbs);
}
public T Get(CommandBufferScoped cbs, int offset, int size, bool write = false)
{
_mirrorable?.ClearMirrors(cbs, offset, size);
_waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size, write);
return Get(cbs); return Get(cbs);
} }

263
src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs Normal file
View file

@ -0,0 +1,263 @@
using Ryujinx.Common.Memory;
using System;
using System.Numerics;
namespace Ryujinx.Graphics.Vulkan
{
interface IBitMapListener
{
void BitMapSignal(int index, int count);
}
struct BitMapStruct<T> where T : IArray<long>
{
public const int IntSize = 64;
private const int IntShift = 6;
private const int IntMask = IntSize - 1;
private T _masks;
public BitMapStruct()
{
_masks = default;
}
public bool BecomesUnsetFrom(in BitMapStruct<T> from, ref BitMapStruct<T> into)
{
bool result = false;
int masks = _masks.Length;
for (int i = 0; i < masks; i++)
{
long fromMask = from._masks[i];
long unsetMask = (~fromMask) & (fromMask ^ _masks[i]);
into._masks[i] = unsetMask;
result |= unsetMask != 0;
}
return result;
}
public void SetAndSignalUnset<T2>(in BitMapStruct<T> from, ref T2 listener) where T2 : struct, IBitMapListener
{
BitMapStruct<T> result = new();
if (BecomesUnsetFrom(from, ref result))
{
// Iterate the set bits in the result, and signal them.
int offset = 0;
int masks = _masks.Length;
ref T resultMasks = ref result._masks;
for (int i = 0; i < masks; i++)
{
long value = resultMasks[i];
while (value != 0)
{
int bit = BitOperations.TrailingZeroCount((ulong)value);
listener.BitMapSignal(offset + bit, 1);
value &= ~(1L << bit);
}
offset += IntSize;
}
}
_masks = from._masks;
}
public void SignalSet(Action<int, int> action)
{
// Iterate the set bits in the result, and signal them.
int offset = 0;
int masks = _masks.Length;
for (int i = 0; i < masks; i++)
{
long value = _masks[i];
while (value != 0)
{
int bit = BitOperations.TrailingZeroCount((ulong)value);
action(offset + bit, 1);
value &= ~(1L << bit);
}
offset += IntSize;
}
}
public bool AnySet()
{
for (int i = 0; i < _masks.Length; i++)
{
if (_masks[i] != 0)
{
return true;
}
}
return false;
}
public bool IsSet(int bit)
{
int wordIndex = bit >> IntShift;
int wordBit = bit & IntMask;
long wordMask = 1L << wordBit;
return (_masks[wordIndex] & wordMask) != 0;
}
public bool IsSet(int start, int end)
{
if (start == end)
{
return IsSet(start);
}
int startIndex = start >> IntShift;
int startBit = start & IntMask;
long startMask = -1L << startBit;
int endIndex = end >> IntShift;
int endBit = end & IntMask;
long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
if (startIndex == endIndex)
{
return (_masks[startIndex] & startMask & endMask) != 0;
}
if ((_masks[startIndex] & startMask) != 0)
{
return true;
}
for (int i = startIndex + 1; i < endIndex; i++)
{
if (_masks[i] != 0)
{
return true;
}
}
if ((_masks[endIndex] & endMask) != 0)
{
return true;
}
return false;
}
public bool Set(int bit)
{
int wordIndex = bit >> IntShift;
int wordBit = bit & IntMask;
long wordMask = 1L << wordBit;
if ((_masks[wordIndex] & wordMask) != 0)
{
return false;
}
_masks[wordIndex] |= wordMask;
return true;
}
public void Set(int bit, bool value)
{
if (value)
{
Set(bit);
}
else
{
Clear(bit);
}
}
public void SetRange(int start, int end)
{
if (start == end)
{
Set(start);
return;
}
int startIndex = start >> IntShift;
int startBit = start & IntMask;
long startMask = -1L << startBit;
int endIndex = end >> IntShift;
int endBit = end & IntMask;
long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
if (startIndex == endIndex)
{
_masks[startIndex] |= startMask & endMask;
}
else
{
_masks[startIndex] |= startMask;
for (int i = startIndex + 1; i < endIndex; i++)
{
_masks[i] |= -1L;
}
_masks[endIndex] |= endMask;
}
}
public BitMapStruct<T> Union(BitMapStruct<T> other)
{
var result = new BitMapStruct<T>();
ref var masks = ref _masks;
ref var otherMasks = ref other._masks;
ref var newMasks = ref result._masks;
for (int i = 0; i < masks.Length; i++)
{
newMasks[i] = masks[i] | otherMasks[i];
}
return result;
}
public void Clear(int bit)
{
int wordIndex = bit >> IntShift;
int wordBit = bit & IntMask;
long wordMask = 1L << wordBit;
_masks[wordIndex] &= ~wordMask;
}
public void Clear()
{
for (int i = 0; i < _masks.Length; i++)
{
_masks[i] = 0;
}
}
public void ClearInt(int start, int end)
{
for (int i = start; i <= end; i++)
{
_masks[i] = 0;
}
}
}
}

283
src/Ryujinx.Graphics.Vulkan/BufferHolder.cs
View file

@ -10,7 +10,7 @@ using VkFormat = Silk.NET.Vulkan.Format;
namespace Ryujinx.Graphics.Vulkan namespace Ryujinx.Graphics.Vulkan
{ {
class BufferHolder : IDisposable class BufferHolder : IDisposable, IMirrorable<DisposableBuffer>, IMirrorable<DisposableBufferView>
{ {
private const int MaxUpdateBufferSize = 0x10000; private const int MaxUpdateBufferSize = 0x10000;
@ -64,6 +64,11 @@ namespace Ryujinx.Graphics.Vulkan
private List<Action> _swapActions; private List<Action> _swapActions;
private byte[] _pendingData;
private BufferMirrorRangeList _pendingDataRanges;
private Dictionary<ulong, StagingBufferReserved> _mirrors;
private bool _useMirrors;
public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size, BufferAllocationType type, BufferAllocationType currentType) public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size, BufferAllocationType type, BufferAllocationType currentType)
{ {
_gd = gd; _gd = gd;
@ -71,7 +76,7 @@ namespace Ryujinx.Graphics.Vulkan
_allocation = allocation; _allocation = allocation;
_allocationAuto = new Auto<MemoryAllocation>(allocation); _allocationAuto = new Auto<MemoryAllocation>(allocation);
_waitable = new MultiFenceHolder(size); _waitable = new MultiFenceHolder(size);
_buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), _waitable, _allocationAuto); _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), this, _waitable, _allocationAuto);
_bufferHandle = buffer.Handle; _bufferHandle = buffer.Handle;
Size = size; Size = size;
_map = allocation.HostPointer; _map = allocation.HostPointer;
@ -81,6 +86,7 @@ namespace Ryujinx.Graphics.Vulkan
DesiredType = currentType; DesiredType = currentType;
_flushLock = new ReaderWriterLock(); _flushLock = new ReaderWriterLock();
_useMirrors = gd.IsTBDR;
} }
public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, Auto<MemoryAllocation> allocation, int size, BufferAllocationType type, BufferAllocationType currentType, int offset) public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, Auto<MemoryAllocation> allocation, int size, BufferAllocationType type, BufferAllocationType currentType, int offset)
@ -91,7 +97,7 @@ namespace Ryujinx.Graphics.Vulkan
_allocationAuto = allocation; _allocationAuto = allocation;
_allocationImported = true; _allocationImported = true;
_waitable = new MultiFenceHolder(size); _waitable = new MultiFenceHolder(size);
_buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), _waitable, _allocationAuto); _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), this, _waitable, _allocationAuto);
_bufferHandle = buffer.Handle; _bufferHandle = buffer.Handle;
Size = size; Size = size;
_map = _allocation.HostPointer + offset; _map = _allocation.HostPointer + offset;
@ -110,7 +116,7 @@ namespace Ryujinx.Graphics.Vulkan
// Only swap if the buffer is not used in any queued command buffer. // Only swap if the buffer is not used in any queued command buffer.
bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool); bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool);
if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld) if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld && (_pendingData == null || cbs != null))
{ {
var currentAllocation = _allocationAuto; var currentAllocation = _allocationAuto;
var currentBuffer = _buffer; var currentBuffer = _buffer;
@ -120,6 +126,11 @@ namespace Ryujinx.Graphics.Vulkan
if (buffer.Handle != 0) if (buffer.Handle != 0)
{ {
if (cbs != null)
{
ClearMirrors(cbs.Value, 0, Size);
}
_flushLock.AcquireWriterLock(Timeout.Infinite); _flushLock.AcquireWriterLock(Timeout.Infinite);
ClearFlushFence(); ClearFlushFence();
@ -128,7 +139,7 @@ namespace Ryujinx.Graphics.Vulkan
_allocation = allocation; _allocation = allocation;
_allocationAuto = new Auto<MemoryAllocation>(allocation); _allocationAuto = new Auto<MemoryAllocation>(allocation);
_buffer = new Auto<DisposableBuffer>(new DisposableBuffer(_gd.Api, _device, buffer), _waitable, _allocationAuto); _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(_gd.Api, _device, buffer), this, _waitable, _allocationAuto);
_bufferHandle = buffer.Handle; _bufferHandle = buffer.Handle;
_map = allocation.HostPointer; _map = allocation.HostPointer;
@ -257,7 +268,7 @@ namespace Ryujinx.Graphics.Vulkan
(_swapActions ??= new List<Action>()).Add(invalidateView); (_swapActions ??= new List<Action>()).Add(invalidateView);
return new Auto<DisposableBufferView>(new DisposableBufferView(_gd.Api, _device, bufferView), _waitable, _buffer); return new Auto<DisposableBufferView>(new DisposableBufferView(_gd.Api, _device, bufferView), this, _waitable, _buffer);
} }
public void InheritMetrics(BufferHolder other) public void InheritMetrics(BufferHolder other)
@ -302,6 +313,82 @@ namespace Ryujinx.Graphics.Vulkan
} }
} }
private static ulong ToMirrorKey(int offset, int size)
{
return ((ulong)offset << 32) | (uint)size;
}
private static (int offset, int size) FromMirrorKey(ulong key)
{
return ((int)(key >> 32), (int)key);
}
private unsafe bool TryGetMirror(CommandBufferScoped cbs, ref int offset, int size, out Auto<DisposableBuffer> buffer)
{
size = Math.Min(size, Size - offset);
// Does this binding need to be mirrored?
if (!_pendingDataRanges.OverlapsWith(offset, size))
{
buffer = null;
return false;
}
var key = ToMirrorKey(offset, size);
if (_mirrors.TryGetValue(key, out StagingBufferReserved reserved))
{
buffer = reserved.Buffer.GetBuffer();
offset = reserved.Offset;
return true;
}
// Is this mirror allowed to exist? Can't be used for write in any in-flight write.
if (_waitable.IsBufferRangeInUse(offset, size, true))
{
// Some of the data is not mirrorable, so upload the whole range.
ClearMirrors(cbs, offset, size);
buffer = null;
return false;
}
// Build data for the new mirror.
var baseData = new Span<byte>((void*)(_map + offset), size);
var modData = _pendingData.AsSpan(offset, size);
StagingBufferReserved? newMirror = _gd.BufferManager.StagingBuffer.TryReserveData(cbs, size, (int)_gd.Capabilities.MinResourceAlignment);
if (newMirror != null)
{
var mirror = newMirror.Value;
_pendingDataRanges.FillData(baseData, modData, offset, new Span<byte>((void*)(mirror.Buffer._map + mirror.Offset), size));
if (_mirrors.Count == 0)
{
_gd.PipelineInternal.RegisterActiveMirror(this);
}
_mirrors.Add(key, mirror);
buffer = mirror.Buffer.GetBuffer();
offset = mirror.Offset;
return true;
}
else
{
// Data could not be placed on the mirror, likely out of space. Force the data to flush.
ClearMirrors(cbs, offset, size);
buffer = null;
return false;
}
}
public Auto<DisposableBuffer> GetBuffer() public Auto<DisposableBuffer> GetBuffer()
{ {
return _buffer; return _buffer;
@ -339,6 +426,86 @@ namespace Ryujinx.Graphics.Vulkan
return _buffer; return _buffer;
} }
public Auto<DisposableBuffer> GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored)
{
if (_pendingData != null && TryGetMirror(cbs, ref offset, size, out Auto<DisposableBuffer> result))
{
mirrored = true;
return result;
}
mirrored = false;
return _buffer;
}
Auto<DisposableBufferView> IMirrorable<DisposableBufferView>.GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored)
{
// Cannot mirror buffer views right now.
throw new NotImplementedException();
}
public void ClearMirrors()
{
// Clear mirrors without forcing a flush. This happens when the command buffer is switched,
// as all reserved areas on the staging buffer are released.
if (_pendingData != null)
{
_mirrors.Clear();
};
}
public void ClearMirrors(CommandBufferScoped cbs, int offset, int size)
{
// Clear mirrors in the given range, and submit overlapping pending data.
if (_pendingData != null)
{
bool hadMirrors = _mirrors.Count > 0 && RemoveOverlappingMirrors(offset, size);
if (_pendingDataRanges.Count() != 0)
{
UploadPendingData(cbs, offset, size);
}
if (hadMirrors)
{
_gd.PipelineInternal.Rebind(_buffer, offset, size);
}
};
}
public void UseMirrors()
{
_useMirrors = true;
}
private void UploadPendingData(CommandBufferScoped cbs, int offset, int size)
{
var ranges = _pendingDataRanges.FindOverlaps(offset, size);
if (ranges != null)
{
_pendingDataRanges.Remove(offset, size);
foreach (var range in ranges)
{
int rangeOffset = Math.Max(offset, range.Offset);
int rangeSize = Math.Min(offset + size, range.End) - rangeOffset;
if (_gd.PipelineInternal.CurrentCommandBuffer.CommandBuffer.Handle == cbs.CommandBuffer.Handle)
{
SetData(rangeOffset, _pendingData.AsSpan(rangeOffset, rangeSize), cbs, _gd.PipelineInternal.EndRenderPass, false);
}
else
{
SetData(rangeOffset, _pendingData.AsSpan(rangeOffset, rangeSize), cbs, null, false);
}
}
}
}
public void SignalWrite(int offset, int size) public void SignalWrite(int offset, int size)
{ {
ConsiderBackingSwap(); ConsiderBackingSwap();
@ -472,7 +639,34 @@ namespace Ryujinx.Graphics.Vulkan
throw new InvalidOperationException("The buffer is not host mapped."); throw new InvalidOperationException("The buffer is not host mapped.");
} }
public unsafe void SetData(int offset, ReadOnlySpan<byte> data, CommandBufferScoped? cbs = null, Action endRenderPass = null) public bool RemoveOverlappingMirrors(int offset, int size)
{
List<ulong> toRemove = null;
foreach (var key in _mirrors.Keys)
{
(int keyOffset, int keySize) = FromMirrorKey(key);
if (!(offset + size <= keyOffset || offset >= keyOffset + keySize))
{
toRemove ??= new List<ulong>();
toRemove.Add(key);
}
}
if (toRemove != null)
{
foreach (var key in toRemove)
{
_mirrors.Remove(key);
}
return true;
}
return false;
}
public unsafe void SetData(int offset, ReadOnlySpan<byte> data, CommandBufferScoped? cbs = null, Action endRenderPass = null, bool allowCbsWait = true)
{ {
int dataSize = Math.Min(data.Length, Size - offset); int dataSize = Math.Min(data.Length, Size - offset);
if (dataSize == 0) if (dataSize == 0)
@ -481,6 +675,7 @@ namespace Ryujinx.Graphics.Vulkan
} }
_setCount++; _setCount++;
bool allowMirror = _useMirrors && allowCbsWait && cbs != null && _currentType <= BufferAllocationType.HostMapped;
if (_map != IntPtr.Zero) if (_map != IntPtr.Zero)
{ {
@ -488,7 +683,7 @@ namespace Ryujinx.Graphics.Vulkan
bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool); bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool);
// If the buffer is rented, take a little more time and check if the use overlaps this handle. // If the buffer is rented, take a little more time and check if the use overlaps this handle.
bool needsFlush = isRented && _waitable.IsBufferRangeInUse(offset, dataSize); bool needsFlush = isRented && _waitable.IsBufferRangeInUse(offset, dataSize, false);
if (!needsFlush) if (!needsFlush)
{ {
@ -496,12 +691,48 @@ namespace Ryujinx.Graphics.Vulkan
data[..dataSize].CopyTo(new Span<byte>((void*)(_map + offset), dataSize)); data[..dataSize].CopyTo(new Span<byte>((void*)(_map + offset), dataSize));
if (_pendingData != null)
{
bool removed = _pendingDataRanges.Remove(offset, dataSize);
if (RemoveOverlappingMirrors(offset, dataSize) || removed)
{
// If any mirrors were removed, rebind the buffer range.
_gd.PipelineInternal.Rebind(_buffer, offset, dataSize);
}
}
SignalWrite(offset, dataSize); SignalWrite(offset, dataSize);
return; return;
} }
} }
// If the buffer does not have an in-flight write (including an inline update), then upload data to a pendingCopy.
if (allowMirror && !_waitable.IsBufferRangeInUse(offset, dataSize, true))
{
if (_pendingData == null)
{
_pendingData = new byte[Size];
_mirrors = new Dictionary<ulong, StagingBufferReserved>();
}
data[..dataSize].CopyTo(_pendingData.AsSpan(offset, dataSize));
_pendingDataRanges.Add(offset, dataSize);
// Remove any overlapping mirrors.
RemoveOverlappingMirrors(offset, dataSize);
// Tell the graphics device to rebind any constant buffer that overlaps the newly modified range, as it should access a mirror.
_gd.PipelineInternal.Rebind(_buffer, offset, dataSize);
return;
}
if (_pendingData != null)
{
_pendingDataRanges.Remove(offset, dataSize);
}
if (cbs != null && if (cbs != null &&
_gd.PipelineInternal.RenderPassActive && _gd.PipelineInternal.RenderPassActive &&
!(_buffer.HasCommandBufferDependency(cbs.Value) && !(_buffer.HasCommandBufferDependency(cbs.Value) &&
@ -519,7 +750,37 @@ namespace Ryujinx.Graphics.Vulkan
data.Length > MaxUpdateBufferSize || data.Length > MaxUpdateBufferSize ||
!TryPushData(cbs.Value, endRenderPass, offset, data)) !TryPushData(cbs.Value, endRenderPass, offset, data))
{ {
_gd.BufferManager.StagingBuffer.PushData(_gd.CommandBufferPool, cbs, endRenderPass, this, offset, data); if (allowCbsWait)
{
_gd.BufferManager.StagingBuffer.PushData(_gd.CommandBufferPool, cbs, endRenderPass, this, offset, data);
}
else
{
bool rentCbs = cbs == null;
if (rentCbs)
{
cbs = _gd.CommandBufferPool.Rent();
}
if (!_gd.BufferManager.StagingBuffer.TryPushData(cbs.Value, endRenderPass, this, offset, data))
{
// Need to do a slow upload.
BufferHolder srcHolder = _gd.BufferManager.Create(_gd, dataSize, baseType: BufferAllocationType.HostMapped);
srcHolder.SetDataUnchecked(0, data);
var srcBuffer = srcHolder.GetBuffer();
var dstBuffer = this.GetBuffer(cbs.Value.CommandBuffer, true);
Copy(_gd, cbs.Value, srcBuffer, dstBuffer, 0, offset, dataSize);
srcHolder.Dispose();
}
if (rentCbs)
{
cbs.Value.Dispose();
}
}
} }
} }
@ -558,7 +819,7 @@ namespace Ryujinx.Graphics.Vulkan
endRenderPass?.Invoke(); endRenderPass?.Invoke();
var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length).Value; var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length, true).Value;
_writeCount--; _writeCount--;
@ -608,7 +869,7 @@ namespace Ryujinx.Graphics.Vulkan
bool registerSrcUsage = true) bool registerSrcUsage = true)
{ {
var srcBuffer = registerSrcUsage ? src.Get(cbs, srcOffset, size).Value : src.GetUnsafe().Value; var srcBuffer = registerSrcUsage ? src.Get(cbs, srcOffset, size).Value : src.GetUnsafe().Value;
var dstBuffer = dst.Get(cbs, dstOffset, size).Value; var dstBuffer = dst.Get(cbs, dstOffset, size, true).Value;
InsertBufferBarrier( InsertBufferBarrier(
gd, gd,

13
src/Ryujinx.Graphics.Vulkan/BufferManager.cs
View file

@ -100,9 +100,10 @@ namespace Ryujinx.Graphics.Vulkan
VulkanRenderer gd, VulkanRenderer gd,
int size, int size,
BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferAllocationType baseType = BufferAllocationType.HostMapped,
BufferHandle storageHint = default) BufferHandle storageHint = default,
bool forceMirrors = false)
{ {
return CreateWithHandle(gd, size, out _, baseType, storageHint); return CreateWithHandle(gd, size, out _, baseType, storageHint, forceMirrors);
} }
public BufferHandle CreateWithHandle( public BufferHandle CreateWithHandle(
@ -110,7 +111,8 @@ namespace Ryujinx.Graphics.Vulkan
int size, int size,
out BufferHolder holder, out BufferHolder holder,
BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferAllocationType baseType = BufferAllocationType.HostMapped,
BufferHandle storageHint = default) BufferHandle storageHint = default,
bool forceMirrors = false)
{ {
holder = Create(gd, size, baseType: baseType, storageHint: storageHint); holder = Create(gd, size, baseType: baseType, storageHint: storageHint);
if (holder == null) if (holder == null)
@ -118,6 +120,11 @@ namespace Ryujinx.Graphics.Vulkan
return BufferHandle.Null; return BufferHandle.Null;
} }
if (forceMirrors)
{
holder.UseMirrors();
}
BufferCount++; BufferCount++;
ulong handle64 = (uint)_buffers.Add(holder); ulong handle64 = (uint)_buffers.Add(holder);

305
src/Ryujinx.Graphics.Vulkan/BufferMirrorRangeList.cs Normal file
View file

@ -0,0 +1,305 @@
using System;
using System.Collections.Generic;
namespace Ryujinx.Graphics.Vulkan
{
/// <summary>
/// A structure tracking pending upload ranges for buffers.
/// Where a range is present, pending data exists that can either be used to build mirrors
/// or upload directly to the buffer.
/// </summary>
struct BufferMirrorRangeList
{
internal readonly struct Range
{
public int Offset { get; }
public int Size { get; }
public int End => Offset + Size;
public Range(int offset, int size)
{
Offset = offset;
Size = size;
}
public bool OverlapsWith(int offset, int size)
{
return Offset < offset + size && offset < Offset + Size;
}
}
private List<Range> _ranges;
public readonly IEnumerable<Range> All()
{
return _ranges;
}
public readonly bool Remove(int offset, int size)
{
var list = _ranges;
bool removedAny = false;
if (list != null)
{
int overlapIndex = BinarySearch(list, offset, size);
if (overlapIndex >= 0)
{
// Overlaps with a range. Search back to find the first one it doesn't overlap with.
while (overlapIndex > 0 && list[overlapIndex - 1].OverlapsWith(offset, size))
{
overlapIndex--;
}
int endOffset = offset + size;
int startIndex = overlapIndex;
var currentOverlap = list[overlapIndex];
// Orphan the start of the overlap.
if (currentOverlap.Offset < offset)
{
list[overlapIndex] = new Range(currentOverlap.Offset, offset - currentOverlap.Offset);
currentOverlap = new Range(offset, currentOverlap.End - offset);
list.Insert(++overlapIndex, currentOverlap);
startIndex++;
removedAny = true;
}
// Remove any middle overlaps.
while (currentOverlap.Offset < endOffset)
{
if (currentOverlap.End > endOffset)
{
// Update the end overlap instead of removing it, if it spans beyond the removed range.
list[overlapIndex] = new Range(endOffset, currentOverlap.End - endOffset);
removedAny = true;
break;
}
if (++overlapIndex >= list.Count)
{
break;
}
currentOverlap = list[overlapIndex];
}
int count = overlapIndex - startIndex;
list.RemoveRange(startIndex, count);
removedAny |= count > 0;
}
}
return removedAny;
}
public void Add(int offset, int size)
{
var list = _ranges;
if (list != null)
{
int overlapIndex = BinarySearch(list, offset, size);
if (overlapIndex >= 0)
{
while (overlapIndex > 0 && list[overlapIndex - 1].OverlapsWith(offset, size))
{
overlapIndex--;
}
int endOffset = offset + size;
int startIndex = overlapIndex;
while (overlapIndex < list.Count && list[overlapIndex].OverlapsWith(offset, size))
{
var currentOverlap = list[overlapIndex];
var currentOverlapEndOffset = currentOverlap.Offset + currentOverlap.Size;
if (offset > currentOverlap.Offset)
{
offset = currentOverlap.Offset;
}
if (endOffset < currentOverlapEndOffset)
{
endOffset = currentOverlapEndOffset;
}
overlapIndex++;
size = endOffset - offset;
}
int count = overlapIndex - startIndex;
list.RemoveRange(startIndex, count);
overlapIndex = startIndex;
}
else
{
overlapIndex = ~overlapIndex;
}
list.Insert(overlapIndex, new Range(offset, size));
}
else
{
_ranges = new List<Range>
{
new Range(offset, size)
};
}
}
public readonly bool OverlapsWith(int offset, int size)
{
var list = _ranges;
if (list == null)
{
return false;
}
return BinarySearch(list, offset, size) >= 0;
}
public readonly List<Range> FindOverlaps(int offset, int size)
{
var list = _ranges;
if (list == null)
{
return null;
}
List<Range> result = null;
int index = BinarySearch(list, offset, size);
if (index >= 0)
{
while (index > 0 && list[index - 1].OverlapsWith(offset, size))
{
index--;
}
do
{
(result ??= new List<Range>()).Add(list[index++]);
}
while (index < list.Count && list[index].OverlapsWith(offset, size));
}
return result;
}
private static int BinarySearch(List<Range> list, int offset, int size)
{
int left = 0;
int right = list.Count - 1;
while (left <= right)
{
int range = right - left;
int middle = left + (range >> 1);
var item = list[middle];
if (item.OverlapsWith(offset, size))
{
return middle;
}
if (offset < item.Offset)
{
right = middle - 1;
}
else
{
left = middle + 1;
}
}
return ~left;
}
public readonly void FillData(Span<byte> baseData, Span<byte> modData, int offset, Span<byte> result)
{
int size = baseData.Length;
int endOffset = offset + size;
var list = _ranges;
if (list == null)
{
baseData.CopyTo(result);
}
int srcOffset = offset;
int dstOffset = 0;
bool activeRange = false;
for (int i = 0; i < list.Count; i++)
{
var range = list[i];
int rangeEnd = range.Offset + range.Size;
if (activeRange)
{
if (range.Offset >= endOffset)
{
break;
}
}
else
{
if (rangeEnd <= offset)
{
continue;
}
activeRange = true;
}
int baseSize = range.Offset - srcOffset;
if (baseSize > 0)
{
baseData.Slice(dstOffset, baseSize).CopyTo(result.Slice(dstOffset, baseSize));
srcOffset += baseSize;
dstOffset += baseSize;
}
int modSize = Math.Min(rangeEnd - srcOffset, endOffset - srcOffset);
if (modSize != 0)
{
modData.Slice(dstOffset, modSize).CopyTo(result.Slice(dstOffset, modSize));
srcOffset += modSize;
dstOffset += modSize;
}
}
int baseSizeEnd = endOffset - srcOffset;
if (baseSizeEnd > 0)
{
baseData.Slice(dstOffset, baseSizeEnd).CopyTo(result.Slice(dstOffset, baseSizeEnd));
}
}
public readonly int Count()
{
return _ranges?.Count ?? 0;
}
public void Clear()
{
_ranges = null;
}
}
}

7
src/Ryujinx.Graphics.Vulkan/BufferState.cs
View file

@ -23,7 +23,7 @@ namespace Ryujinx.Graphics.Vulkan
{ {
if (_buffer != null) if (_buffer != null)
{ {
var buffer = _buffer.Get(cbs, _offset, _size).Value; var buffer = _buffer.Get(cbs, _offset, _size, true).Value;
gd.TransformFeedbackApi.CmdBindTransformFeedbackBuffers(cbs.CommandBuffer, binding, 1, buffer, (ulong)_offset, (ulong)_size); gd.TransformFeedbackApi.CmdBindTransformFeedbackBuffers(cbs.CommandBuffer, binding, 1, buffer, (ulong)_offset, (ulong)_size);
} }
@ -40,6 +40,11 @@ namespace Ryujinx.Graphics.Vulkan
} }
} }
public readonly bool Overlaps(Auto<DisposableBuffer> buffer, int offset, int size)
{
return buffer == _buffer && offset < _offset + _size && offset + size > _offset;
}
public readonly void Dispose() public readonly void Dispose()
{ {
_buffer?.DecrementReferenceCount(); _buffer?.DecrementReferenceCount();

19
src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs
View file

@ -6,6 +6,7 @@
private readonly int _size; private readonly int _size;
private readonly int _granularity; private readonly int _granularity;
private readonly int _bits; private readonly int _bits;
private readonly int _writeBitOffset;
private readonly int _intsPerCb; private readonly int _intsPerCb;
private readonly int _bitsPerCb; private readonly int _bitsPerCb;
@ -14,7 +15,11 @@
{ {
_size = size; _size = size;
_granularity = granularity; _granularity = granularity;
_bits = (size + (granularity - 1)) / granularity;
// There are two sets of bits - one for read tracking, and the other for write.
int bits = (size + (granularity - 1)) / granularity;
_writeBitOffset = bits;
_bits = bits << 1;
_intsPerCb = (_bits + (BitMap.IntSize - 1)) / BitMap.IntSize; _intsPerCb = (_bits + (BitMap.IntSize - 1)) / BitMap.IntSize;
_bitsPerCb = _intsPerCb * BitMap.IntSize; _bitsPerCb = _intsPerCb * BitMap.IntSize;
@ -22,7 +27,7 @@
_bitmap = new BitMap(_bitsPerCb * CommandBufferPool.MaxCommandBuffers); _bitmap = new BitMap(_bitsPerCb * CommandBufferPool.MaxCommandBuffers);
} }
public void Add(int cbIndex, int offset, int size) public void Add(int cbIndex, int offset, int size, bool write)
{ {
if (size == 0) if (size == 0)
{ {
@ -35,32 +40,32 @@
size = _size - offset; size = _size - offset;
} }
int cbBase = cbIndex * _bitsPerCb; int cbBase = cbIndex * _bitsPerCb + (write ? _writeBitOffset : 0);
int start = cbBase + offset / _granularity; int start = cbBase + offset / _granularity;
int end = cbBase + (offset + size - 1) / _granularity; int end = cbBase + (offset + size - 1) / _granularity;
_bitmap.SetRange(start, end); _bitmap.SetRange(start, end);
} }
public bool OverlapsWith(int cbIndex, int offset, int size) public bool OverlapsWith(int cbIndex, int offset, int size, bool write = false)
{ {
if (size == 0) if (size == 0)
{ {
return false; return false;
} }
int cbBase = cbIndex * _bitsPerCb; int cbBase = cbIndex * _bitsPerCb + (write ? _writeBitOffset : 0);
int start = cbBase + offset / _granularity; int start = cbBase + offset / _granularity;
int end = cbBase + (offset + size - 1) / _granularity; int end = cbBase + (offset + size - 1) / _granularity;
return _bitmap.IsSet(start, end); return _bitmap.IsSet(start, end);
} }
public bool OverlapsWith(int offset, int size) public bool OverlapsWith(int offset, int size, bool write)
{ {
for (int i = 0; i < CommandBufferPool.MaxCommandBuffers; i++) for (int i = 0; i < CommandBufferPool.MaxCommandBuffers; i++)
{ {
if (OverlapsWith(i, offset, size)) if (OverlapsWith(i, offset, size, write))
{ {
return true; return true;
} }

211
src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs
View file

@ -1,9 +1,9 @@
using Ryujinx.Graphics.GAL; using Ryujinx.Common.Memory;
using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Shader; using Ryujinx.Graphics.Shader;
using Silk.NET.Vulkan; using Silk.NET.Vulkan;
using System; using System;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using Buffer = Silk.NET.Vulkan.Buffer;
using CompareOp = Ryujinx.Graphics.GAL.CompareOp; using CompareOp = Ryujinx.Graphics.GAL.CompareOp;
using Format = Ryujinx.Graphics.GAL.Format; using Format = Ryujinx.Graphics.GAL.Format;
using SamplerCreateInfo = Ryujinx.Graphics.GAL.SamplerCreateInfo; using SamplerCreateInfo = Ryujinx.Graphics.GAL.SamplerCreateInfo;
@ -12,13 +12,34 @@ namespace Ryujinx.Graphics.Vulkan
{ {
class DescriptorSetUpdater class DescriptorSetUpdater
{ {
private const ulong StorageBufferMaxMirrorable = 0x2000;
private record struct BufferRef
{
public Auto<DisposableBuffer> Buffer;
public int Offset;
public bool Write;
public BufferRef(Auto<DisposableBuffer> buffer)
{
Buffer = buffer;
Offset = 0;
Write = true;
}
public BufferRef(Auto<DisposableBuffer> buffer, ref BufferRange range)
{
Buffer = buffer;
Offset = range.Offset;
Write = range.Write;
}
}
private readonly VulkanRenderer _gd; private readonly VulkanRenderer _gd;
private readonly PipelineBase _pipeline; private readonly PipelineBase _pipeline;
private ShaderCollection _program; private ShaderCollection _program;
private readonly Auto<DisposableBuffer>[] _uniformBufferRefs; private readonly BufferRef[] _uniformBufferRefs;
private readonly Auto<DisposableBuffer>[] _storageBufferRefs; private readonly BufferRef[] _storageBufferRefs;
private readonly Auto<DisposableImageView>[] _textureRefs; private readonly Auto<DisposableImageView>[] _textureRefs;
private readonly Auto<DisposableSampler>[] _samplerRefs; private readonly Auto<DisposableSampler>[] _samplerRefs;
private readonly Auto<DisposableImageView>[] _imageRefs; private readonly Auto<DisposableImageView>[] _imageRefs;
@ -33,8 +54,10 @@ namespace Ryujinx.Graphics.Vulkan
private readonly BufferView[] _bufferTextures; private readonly BufferView[] _bufferTextures;
private readonly BufferView[] _bufferImages; private readonly BufferView[] _bufferImages;
private readonly bool[] _uniformSet; private BitMapStruct<Array2<long>> _uniformSet;
private readonly bool[] _storageSet; private BitMapStruct<Array2<long>> _storageSet;
private BitMapStruct<Array2<long>> _uniformMirrored;
private BitMapStruct<Array2<long>> _storageMirrored;
[Flags] [Flags]
private enum DirtyFlags private enum DirtyFlags
@ -61,8 +84,8 @@ namespace Ryujinx.Graphics.Vulkan
// Some of the bindings counts needs to be multiplied by 2 because we have buffer and // Some of the bindings counts needs to be multiplied by 2 because we have buffer and
// regular textures/images interleaved on the same descriptor set. // regular textures/images interleaved on the same descriptor set.
_uniformBufferRefs = new Auto<DisposableBuffer>[Constants.MaxUniformBufferBindings]; _uniformBufferRefs = new BufferRef[Constants.MaxUniformBufferBindings];
_storageBufferRefs = new Auto<DisposableBuffer>[Constants.MaxStorageBufferBindings]; _storageBufferRefs = new BufferRef[Constants.MaxStorageBufferBindings];
_textureRefs = new Auto<DisposableImageView>[Constants.MaxTextureBindings * 2]; _textureRefs = new Auto<DisposableImageView>[Constants.MaxTextureBindings * 2];
_samplerRefs = new Auto<DisposableSampler>[Constants.MaxTextureBindings * 2]; _samplerRefs = new Auto<DisposableSampler>[Constants.MaxTextureBindings * 2];
_imageRefs = new Auto<DisposableImageView>[Constants.MaxImageBindings * 2]; _imageRefs = new Auto<DisposableImageView>[Constants.MaxImageBindings * 2];
@ -85,9 +108,6 @@ namespace Ryujinx.Graphics.Vulkan
_textures.AsSpan().Fill(initialImageInfo); _textures.AsSpan().Fill(initialImageInfo);
_images.AsSpan().Fill(initialImageInfo); _images.AsSpan().Fill(initialImageInfo);
_uniformSet = new bool[Constants.MaxUniformBufferBindings];
_storageSet = new bool[Constants.MaxStorageBufferBindings];
if (gd.Capabilities.SupportsNullDescriptors) if (gd.Capabilities.SupportsNullDescriptors)
{ {
// If null descriptors are supported, we can pass null as the handle. // If null descriptors are supported, we can pass null as the handle.
@ -138,6 +158,63 @@ namespace Ryujinx.Graphics.Vulkan
_dummyTexture.SetData(dummyTextureData); _dummyTexture.SetData(dummyTextureData);
} }
private static bool BindingOverlaps(ref DescriptorBufferInfo info, int bindingOffset, int offset, int size)
{
return offset < bindingOffset + (int)info.Range && (offset + size) > bindingOffset;
}
internal void Rebind(Auto<DisposableBuffer> buffer, int offset, int size)
{
if (_program == null)
{
return;
}
// Check stage bindings
_uniformMirrored.Union(_uniformSet).SignalSet((int binding, int count) =>
{
for (int i = 0; i < count; i++)
{
ref BufferRef bufferRef = ref _uniformBufferRefs[binding];
if (bufferRef.Buffer == buffer)
{
ref DescriptorBufferInfo info = ref _uniformBuffers[binding];
int bindingOffset = bufferRef.Offset;
if (BindingOverlaps(ref info, bindingOffset, offset, size))
{
_uniformSet.Clear(binding);
SignalDirty(DirtyFlags.Uniform);
}
}
binding++;
}
});
_storageMirrored.Union(_storageSet).SignalSet((int binding, int count) =>
{
for (int i = 0; i < count; i++)
{
ref BufferRef bufferRef = ref _storageBufferRefs[binding];
if (bufferRef.Buffer == buffer)
{
ref DescriptorBufferInfo info = ref _storageBuffers[binding];
int bindingOffset = bufferRef.Offset;
if (BindingOverlaps(ref info, bindingOffset, offset, size))
{
_storageSet.Clear(binding);
SignalDirty(DirtyFlags.Storage);
}
}
binding++;
}
});
}
public void SetProgram(ShaderCollection program) public void SetProgram(ShaderCollection program)
{ {
_program = program; _program = program;
@ -180,22 +257,28 @@ namespace Ryujinx.Graphics.Vulkan
var buffer = assignment.Range; var buffer = assignment.Range;
int index = assignment.Binding; int index = assignment.Binding;
Auto<DisposableBuffer> vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false, isSSBO: true); Auto<DisposableBuffer> vkBuffer = buffer.Handle == BufferHandle.Null
ref Auto<DisposableBuffer> currentVkBuffer = ref _storageBufferRefs[index]; ? null
: _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, buffer.Write, isSSBO: true);
ref BufferRef currentBufferRef = ref _storageBufferRefs[index];
DescriptorBufferInfo info = new() DescriptorBufferInfo info = new()
{ {
Offset = (ulong)buffer.Offset, Offset = (ulong)buffer.Offset,
Range = (ulong)buffer.Size, Range = (ulong)buffer.Size,
}; };
var newRef = new BufferRef(vkBuffer, ref buffer);
ref DescriptorBufferInfo currentInfo = ref _storageBuffers[index]; ref DescriptorBufferInfo currentInfo = ref _storageBuffers[index];
if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range)
{ {
_storageSet[index] = false; _storageSet.Clear(index);
currentInfo = info; currentInfo = info;
currentVkBuffer = vkBuffer; currentBufferRef = newRef;
} }
} }
@ -209,21 +292,24 @@ namespace Ryujinx.Graphics.Vulkan
var vkBuffer = buffers[i]; var vkBuffer = buffers[i];
int index = first + i; int index = first + i;
ref Auto<DisposableBuffer> currentVkBuffer = ref _storageBufferRefs[index]; ref BufferRef currentBufferRef = ref _storageBufferRefs[index];
DescriptorBufferInfo info = new() DescriptorBufferInfo info = new()
{ {
Offset = 0, Offset = 0,
Range = Vk.WholeSize, Range = Vk.WholeSize,
}; };
BufferRef newRef = new(vkBuffer);
ref DescriptorBufferInfo currentInfo = ref _storageBuffers[index]; ref DescriptorBufferInfo currentInfo = ref _storageBuffers[index];
if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range)
{ {
_storageSet[index] = false; _storageSet.Clear(index);
currentInfo = info; currentInfo = info;
currentVkBuffer = vkBuffer; currentBufferRef = newRef;
} }
} }
@ -288,22 +374,28 @@ namespace Ryujinx.Graphics.Vulkan
var buffer = assignment.Range; var buffer = assignment.Range;
int index = assignment.Binding; int index = assignment.Binding;
Auto<DisposableBuffer> vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false); Auto<DisposableBuffer> vkBuffer = buffer.Handle == BufferHandle.Null
ref Auto<DisposableBuffer> currentVkBuffer = ref _uniformBufferRefs[index]; ? null
: _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false);
ref BufferRef currentBufferRef = ref _uniformBufferRefs[index];
DescriptorBufferInfo info = new() DescriptorBufferInfo info = new()
{ {
Offset = (ulong)buffer.Offset, Offset = (ulong)buffer.Offset,
Range = (ulong)buffer.Size, Range = (ulong)buffer.Size,
}; };
BufferRef newRef = new(vkBuffer, ref buffer);
ref DescriptorBufferInfo currentInfo = ref _uniformBuffers[index]; ref DescriptorBufferInfo currentInfo = ref _uniformBuffers[index];
if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range)
{ {
_uniformSet[index] = false; _uniformSet.Clear(index);
currentInfo = info; currentInfo = info;
currentVkBuffer = vkBuffer; currentBufferRef = newRef;
} }
} }
@ -353,13 +445,26 @@ namespace Ryujinx.Graphics.Vulkan
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void UpdateBuffer( private static bool UpdateBuffer(
CommandBufferScoped cbs, CommandBufferScoped cbs,
ref DescriptorBufferInfo info, ref DescriptorBufferInfo info,
Auto<DisposableBuffer> buffer, ref BufferRef buffer,
Auto<DisposableBuffer> dummyBuffer) Auto<DisposableBuffer> dummyBuffer,
bool mirrorable)
{ {
info.Buffer = buffer?.Get(cbs, (int)info.Offset, (int)info.Range).Value ?? default; int offset = buffer.Offset;
bool mirrored = false;
if (mirrorable)
{
info.Buffer = buffer.Buffer?.GetMirrorable(cbs, ref offset, (int)info.Range, out mirrored).Value ?? default;
}
else
{
info.Buffer = buffer.Buffer?.Get(cbs, offset, (int)info.Range, buffer.Write).Value ?? default;
}
info.Offset = (ulong)offset;
// The spec requires that buffers with null handle have offset as 0 and range as VK_WHOLE_SIZE. // The spec requires that buffers with null handle have offset as 0 and range as VK_WHOLE_SIZE.
if (info.Buffer.Handle == 0) if (info.Buffer.Handle == 0)
@ -368,6 +473,8 @@ namespace Ryujinx.Graphics.Vulkan
info.Offset = 0; info.Offset = 0;
info.Range = Vk.WholeSize; info.Range = Vk.WholeSize;
} }
return mirrored;
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
@ -404,11 +511,13 @@ namespace Ryujinx.Graphics.Vulkan
{ {
int index = binding + i; int index = binding + i;
if (!_uniformSet[index]) if (_uniformSet.Set(index))
{ {
UpdateBuffer(cbs, ref _uniformBuffers[index], _uniformBufferRefs[index], dummyBuffer); ref BufferRef buffer = ref _uniformBufferRefs[index];
_uniformSet[index] = true; bool mirrored = UpdateBuffer(cbs, ref _uniformBuffers[index], ref buffer, dummyBuffer, true);
_uniformMirrored.Set(index, mirrored);
} }
} }
@ -421,11 +530,19 @@ namespace Ryujinx.Graphics.Vulkan
{ {
int index = binding + i; int index = binding + i;
if (!_storageSet[index]) ref BufferRef buffer = ref _storageBufferRefs[index];
{
UpdateBuffer(cbs, ref _storageBuffers[index], _storageBufferRefs[index], dummyBuffer);
_storageSet[index] = true; if (_storageSet.Set(index))
{
ref var info = ref _storageBuffers[index];
bool mirrored = UpdateBuffer(cbs,
ref info,
ref _storageBufferRefs[index],
dummyBuffer,
!buffer.Write && info.Range <= StorageBufferMaxMirrorable);
_storageMirrored.Set(index, mirrored);
} }
} }
@ -464,7 +581,7 @@ namespace Ryujinx.Graphics.Vulkan
for (int i = 0; i < count; i++) for (int i = 0; i < count; i++)
{ {
bufferTextures[i] = _bufferTextureRefs[binding + i]?.GetBufferView(cbs) ?? default; bufferTextures[i] = _bufferTextureRefs[binding + i]?.GetBufferView(cbs, false) ?? default;
} }
dsc.UpdateBufferImages(0, binding, bufferTextures[..count], DescriptorType.UniformTexelBuffer); dsc.UpdateBufferImages(0, binding, bufferTextures[..count], DescriptorType.UniformTexelBuffer);
@ -489,7 +606,7 @@ namespace Ryujinx.Graphics.Vulkan
for (int i = 0; i < count; i++) for (int i = 0; i < count; i++)
{ {
bufferImages[i] = _bufferImageRefs[binding + i]?.GetBufferView(cbs, _bufferImageFormats[binding + i]) ?? default; bufferImages[i] = _bufferImageRefs[binding + i]?.GetBufferView(cbs, _bufferImageFormats[binding + i], true) ?? default;
} }
dsc.UpdateBufferImages(0, binding, bufferImages[..count], DescriptorType.StorageTexelBuffer); dsc.UpdateBufferImages(0, binding, bufferImages[..count], DescriptorType.StorageTexelBuffer);
@ -546,10 +663,10 @@ namespace Ryujinx.Graphics.Vulkan
{ {
int index = binding + i; int index = binding + i;
if (!_uniformSet[index]) if (_uniformSet.Set(index))
{ {
UpdateBuffer(cbs, ref _uniformBuffers[index], _uniformBufferRefs[index], dummyBuffer); ref BufferRef buffer = ref _uniformBufferRefs[index];
_uniformSet[index] = true; UpdateBuffer(cbs, ref _uniformBuffers[index], ref buffer, dummyBuffer, true);
doUpdate = true; doUpdate = true;
} }
} }
@ -582,17 +699,17 @@ namespace Ryujinx.Graphics.Vulkan
{ {
_dirty = DirtyFlags.All; _dirty = DirtyFlags.All;
Array.Clear(_uniformSet); _uniformSet.Clear();
Array.Clear(_storageSet); _storageSet.Clear();
} }
private static void SwapBuffer(Auto<DisposableBuffer>[] list, Auto<DisposableBuffer> from, Auto<DisposableBuffer> to) private static void SwapBuffer(BufferRef[] list, Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{ {
for (int i = 0; i < list.Length; i++) for (int i = 0; i < list.Length; i++)
{ {
if (list[i] == from) if (list[i].Buffer == from)
{ {
list[i] = to; list[i].Buffer = to;
} }
} }
} }

1
src/Ryujinx.Graphics.Vulkan/EnumConversion.cs
View file

@ -427,6 +427,7 @@ namespace Ryujinx.Graphics.Vulkan
return access switch return access switch
{ {
BufferAccess.FlushPersistent => BufferAllocationType.HostMapped, BufferAccess.FlushPersistent => BufferAllocationType.HostMapped,
BufferAccess.Stream => BufferAllocationType.HostMapped,
_ => BufferAllocationType.Auto, _ => BufferAllocationType.Auto,
}; };
} }

5
src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs
View file

@ -52,6 +52,7 @@ namespace Ryujinx.Graphics.Vulkan
public readonly PortabilitySubsetFlags PortabilitySubset; public readonly PortabilitySubsetFlags PortabilitySubset;
public readonly uint VertexBufferAlignment; public readonly uint VertexBufferAlignment;
public readonly uint SubTexelPrecisionBits; public readonly uint SubTexelPrecisionBits;
public readonly ulong MinResourceAlignment;
public HardwareCapabilities( public HardwareCapabilities(
bool supportsIndexTypeUint8, bool supportsIndexTypeUint8,
@ -89,7 +90,8 @@ namespace Ryujinx.Graphics.Vulkan
SampleCountFlags supportedSampleCounts, SampleCountFlags supportedSampleCounts,
PortabilitySubsetFlags portabilitySubset, PortabilitySubsetFlags portabilitySubset,
uint vertexBufferAlignment, uint vertexBufferAlignment,
uint subTexelPrecisionBits) uint subTexelPrecisionBits,
ulong minResourceAlignment)
{ {
SupportsIndexTypeUint8 = supportsIndexTypeUint8; SupportsIndexTypeUint8 = supportsIndexTypeUint8;
SupportsCustomBorderColor = supportsCustomBorderColor; SupportsCustomBorderColor = supportsCustomBorderColor;
@ -127,6 +129,7 @@ namespace Ryujinx.Graphics.Vulkan
PortabilitySubset = portabilitySubset; PortabilitySubset = portabilitySubset;
VertexBufferAlignment = vertexBufferAlignment; VertexBufferAlignment = vertexBufferAlignment;
SubTexelPrecisionBits = subTexelPrecisionBits; SubTexelPrecisionBits = subTexelPrecisionBits;
MinResourceAlignment = minResourceAlignment;
} }
} }
} }

14
src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs
View file

@ -5,6 +5,8 @@ namespace Ryujinx.Graphics.Vulkan
{ {
internal struct IndexBufferState internal struct IndexBufferState
{ {
private const int IndexBufferMaxMirrorable = 0x20000;
public static IndexBufferState Null => new(BufferHandle.Null, 0, 0); public static IndexBufferState Null => new(BufferHandle.Null, 0, 0);
private readonly int _offset; private readonly int _offset;
@ -37,6 +39,7 @@ namespace Ryujinx.Graphics.Vulkan
Auto<DisposableBuffer> autoBuffer; Auto<DisposableBuffer> autoBuffer;
int offset, size; int offset, size;
IndexType type = _type; IndexType type = _type;
bool mirrorable = false;
if (_type == IndexType.Uint8Ext && !gd.Capabilities.SupportsIndexTypeUint8) if (_type == IndexType.Uint8Ext && !gd.Capabilities.SupportsIndexTypeUint8)
{ {
@ -56,6 +59,8 @@ namespace Ryujinx.Graphics.Vulkan
autoBuffer = null; autoBuffer = null;
} }
mirrorable = _size < IndexBufferMaxMirrorable;
offset = _offset; offset = _offset;
size = _size; size = _size;
} }
@ -64,7 +69,9 @@ namespace Ryujinx.Graphics.Vulkan
if (autoBuffer != null) if (autoBuffer != null)
{ {
gd.Api.CmdBindIndexBuffer(cbs.CommandBuffer, autoBuffer.Get(cbs, offset, size).Value, (ulong)offset, type); DisposableBuffer buffer = mirrorable ? autoBuffer.GetMirrorable(cbs, ref offset, size, out _) : autoBuffer.Get(cbs, offset, size);
gd.Api.CmdBindIndexBuffer(cbs.CommandBuffer, buffer.Value, (ulong)offset, type);
} }
} }
@ -155,5 +162,10 @@ namespace Ryujinx.Graphics.Vulkan
_buffer = to; _buffer = to;
} }
} }
public readonly bool Overlaps(Auto<DisposableBuffer> buffer, int offset, int size)
{
return buffer == _buffer && offset < _offset + _size && offset + size > _offset;
}
} }
} }

17
src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs
View file

@ -32,14 +32,20 @@ namespace Ryujinx.Graphics.Vulkan
} }
/// <summary> /// <summary>
/// Adds buffer usage information to the uses list. /// Adds read/write buffer usage information to the uses list.
/// </summary> /// </summary>
/// <param name="cbIndex">Index of the command buffer where the buffer is used</param> /// <param name="cbIndex">Index of the command buffer where the buffer is used</param>
/// <param name="offset">Offset of the buffer being used</param> /// <param name="offset">Offset of the buffer being used</param>
/// <param name="size">Size of the buffer region being used, in bytes</param> /// <param name="size">Size of the buffer region being used, in bytes</param>
public void AddBufferUse(int cbIndex, int offset, int size) /// <param name="write">Whether the access is a write or not</param>
public void AddBufferUse(int cbIndex, int offset, int size, bool write)
{ {
_bufferUsageBitmap.Add(cbIndex, offset, size); _bufferUsageBitmap.Add(cbIndex, offset, size, false);
if (write)
{
_bufferUsageBitmap.Add(cbIndex, offset, size, true);
}
} }
/// <summary> /// <summary>
@ -68,10 +74,11 @@ namespace Ryujinx.Graphics.Vulkan
/// </summary> /// </summary>
/// <param name="offset">Offset of the buffer being used</param> /// <param name="offset">Offset of the buffer being used</param>
/// <param name="size">Size of the buffer region being used, in bytes</param> /// <param name="size">Size of the buffer region being used, in bytes</param>
/// <param name="write">True if only write usages should count</param>
/// <returns>True if in use, false otherwise</returns> /// <returns>True if in use, false otherwise</returns>
public bool IsBufferRangeInUse(int offset, int size) public bool IsBufferRangeInUse(int offset, int size, bool write)
{ {
return _bufferUsageBitmap.OverlapsWith(offset, size); return _bufferUsageBitmap.OverlapsWith(offset, size, write);
} }
/// <summary> /// <summary>

70
src/Ryujinx.Graphics.Vulkan/PipelineBase.cs
View file

@ -193,7 +193,7 @@ namespace Ryujinx.Graphics.Vulkan
{ {
EndRenderPass(); EndRenderPass();
var dst = Gd.BufferManager.GetBuffer(CommandBuffer, destination, offset, size, true).Get(Cbs, offset, size).Value; var dst = Gd.BufferManager.GetBuffer(CommandBuffer, destination, offset, size, true).Get(Cbs, offset, size, true).Value;
BufferHolder.InsertBufferBarrier( BufferHolder.InsertBufferBarrier(
Gd, Gd,
@ -469,6 +469,10 @@ namespace Ryujinx.Graphics.Vulkan
return; return;
} }
var buffer = Gd.BufferManager
.GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false)
.Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value;
UpdateIndexBufferPattern(); UpdateIndexBufferPattern();
RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); RecreatePipelineIfNeeded(PipelineBindPoint.Graphics);
BeginRenderPass(); BeginRenderPass();
@ -498,10 +502,6 @@ namespace Ryujinx.Graphics.Vulkan
} }
else else
{ {
var buffer = Gd.BufferManager
.GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false)
.Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value;
ResumeTransformFeedbackInternal(); ResumeTransformFeedbackInternal();
Gd.Api.CmdDrawIndexedIndirect(CommandBuffer, buffer, (ulong)indirectBuffer.Offset, 1, (uint)indirectBuffer.Size); Gd.Api.CmdDrawIndexedIndirect(CommandBuffer, buffer, (ulong)indirectBuffer.Offset, 1, (uint)indirectBuffer.Size);
@ -515,15 +515,19 @@ namespace Ryujinx.Graphics.Vulkan
return; return;
} }
var countBuffer = Gd.BufferManager
.GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false)
.Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value;
var buffer = Gd.BufferManager
.GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false)
.Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value;
UpdateIndexBufferPattern(); UpdateIndexBufferPattern();
RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); RecreatePipelineIfNeeded(PipelineBindPoint.Graphics);
BeginRenderPass(); BeginRenderPass();
DrawCount++; DrawCount++;
var countBuffer = Gd.BufferManager
.GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false)
.Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value;
if (_indexBufferPattern != null) if (_indexBufferPattern != null)
{ {
// Convert the index buffer into a supported topology. // Convert the index buffer into a supported topology.
@ -570,10 +574,6 @@ namespace Ryujinx.Graphics.Vulkan
} }
else else
{ {
var buffer = Gd.BufferManager
.GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false)
.Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value;
ResumeTransformFeedbackInternal(); ResumeTransformFeedbackInternal();
if (Gd.Capabilities.SupportsIndirectParameters) if (Gd.Capabilities.SupportsIndirectParameters)
@ -609,15 +609,15 @@ namespace Ryujinx.Graphics.Vulkan
// TODO: Support quads and other unsupported topologies. // TODO: Support quads and other unsupported topologies.
var buffer = Gd.BufferManager
.GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false)
.Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size, false).Value;
RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); RecreatePipelineIfNeeded(PipelineBindPoint.Graphics);
BeginRenderPass(); BeginRenderPass();
ResumeTransformFeedbackInternal(); ResumeTransformFeedbackInternal();
DrawCount++; DrawCount++;
var buffer = Gd.BufferManager
.GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false)
.Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value;
Gd.Api.CmdDrawIndirect(CommandBuffer, buffer, (ulong)indirectBuffer.Offset, 1, (uint)indirectBuffer.Size); Gd.Api.CmdDrawIndirect(CommandBuffer, buffer, (ulong)indirectBuffer.Offset, 1, (uint)indirectBuffer.Size);
} }
@ -634,6 +634,14 @@ namespace Ryujinx.Graphics.Vulkan
return; return;
} }
var buffer = Gd.BufferManager
.GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false)
.Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size, false).Value;
var countBuffer = Gd.BufferManager
.GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false)
.Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size, false).Value;
// TODO: Support quads and other unsupported topologies. // TODO: Support quads and other unsupported topologies.
RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); RecreatePipelineIfNeeded(PipelineBindPoint.Graphics);
@ -641,14 +649,6 @@ namespace Ryujinx.Graphics.Vulkan
ResumeTransformFeedbackInternal(); ResumeTransformFeedbackInternal();
DrawCount++; DrawCount++;
var buffer = Gd.BufferManager
.GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false)
.Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value;
var countBuffer = Gd.BufferManager
.GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false)
.Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value;
Gd.DrawIndirectCountApi.CmdDrawIndirectCount( Gd.DrawIndirectCountApi.CmdDrawIndirectCount(
CommandBuffer, CommandBuffer,
buffer, buffer,
@ -709,6 +709,26 @@ namespace Ryujinx.Graphics.Vulkan
return CommandBuffer.Handle == cb.Handle; return CommandBuffer.Handle == cb.Handle;
} }
internal void Rebind(Auto<DisposableBuffer> buffer, int offset, int size)
{
_descriptorSetUpdater.Rebind(buffer, offset, size);
if (_indexBuffer.Overlaps(buffer, offset, size))
{
_indexBuffer.BindIndexBuffer(Gd, Cbs);
}
for (int i = 0; i < _vertexBuffers.Length; i++)
{
if (_vertexBuffers[i].Overlaps(buffer, offset, size))
{
_vertexBuffers[i].BindVertexBuffer(Gd, Cbs, (uint)i, ref _newState, _vertexBufferUpdater);
}
}
_vertexBufferUpdater.Commit(Cbs);
}
#pragma warning disable CA1822 // Mark member as static #pragma warning disable CA1822 // Mark member as static
public void SetAlphaTest(bool enable, float reference, CompareOp op) public void SetAlphaTest(bool enable, float reference, CompareOp op)
{ {

13
src/Ryujinx.Graphics.Vulkan/PipelineFull.cs
View file

@ -14,6 +14,7 @@ namespace Ryujinx.Graphics.Vulkan
private CounterQueueEvent _activeConditionalRender; private CounterQueueEvent _activeConditionalRender;
private readonly List<BufferedQuery> _pendingQueryCopies; private readonly List<BufferedQuery> _pendingQueryCopies;
private readonly List<BufferHolder> _activeBufferMirrors;
private ulong _byteWeight; private ulong _byteWeight;
@ -24,6 +25,7 @@ namespace Ryujinx.Graphics.Vulkan
_activeQueries = new List<(QueryPool, bool)>(); _activeQueries = new List<(QueryPool, bool)>();
_pendingQueryCopies = new(); _pendingQueryCopies = new();
_backingSwaps = new(); _backingSwaps = new();
_activeBufferMirrors = new();
CommandBuffer = (Cbs = gd.CommandBufferPool.Rent()).CommandBuffer; CommandBuffer = (Cbs = gd.CommandBufferPool.Rent()).CommandBuffer;
} }
@ -233,6 +235,12 @@ namespace Ryujinx.Graphics.Vulkan
Gd.RegisterFlush(); Gd.RegisterFlush();
// Restore per-command buffer state. // Restore per-command buffer state.
foreach (BufferHolder buffer in _activeBufferMirrors)
{
buffer.ClearMirrors();
}
_activeBufferMirrors.Clear();
foreach ((var queryPool, var isOcclusion) in _activeQueries) foreach ((var queryPool, var isOcclusion) in _activeQueries)
{ {
@ -249,6 +257,11 @@ namespace Ryujinx.Graphics.Vulkan
Restore(); Restore();
} }
public void RegisterActiveMirror(BufferHolder buffer)
{
_activeBufferMirrors.Add(buffer);
}
public void BeginQuery(BufferedQuery query, QueryPool pool, bool needsReset, bool isOcclusion, bool fromSamplePool) public void BeginQuery(BufferedQuery query, QueryPool pool, bool needsReset, bool isOcclusion, bool fromSamplePool)
{ {
if (needsReset) if (needsReset)

2
src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs
View file

@ -183,7 +183,7 @@ namespace Ryujinx.Graphics.Vulkan.Queries
public void PoolCopy(CommandBufferScoped cbs) public void PoolCopy(CommandBufferScoped cbs)
{ {
var buffer = _buffer.GetBuffer(cbs.CommandBuffer, true).Get(cbs, 0, sizeof(long)).Value; var buffer = _buffer.GetBuffer(cbs.CommandBuffer, true).Get(cbs, 0, sizeof(long), true).Value;
QueryResultFlags flags = QueryResultFlags.ResultWaitBit; QueryResultFlags flags = QueryResultFlags.ResultWaitBit;

90
src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs
View file

@ -1,12 +1,28 @@
using Ryujinx.Common;
using Ryujinx.Common.Logging;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
namespace Ryujinx.Graphics.Vulkan namespace Ryujinx.Graphics.Vulkan
{ {
readonly struct StagingBufferReserved
{
public readonly BufferHolder Buffer;
public readonly int Offset;
public readonly int Size;
public StagingBufferReserved(BufferHolder buffer, int offset, int size)
{
Buffer = buffer;
Offset = offset;
Size = size;
}
}
class StagingBuffer : IDisposable class StagingBuffer : IDisposable
{ {
private const int BufferSize = 16 * 1024 * 1024; private const int BufferSize = 32 * 1024 * 1024;
private int _freeOffset; private int _freeOffset;
private int _freeSize; private int _freeSize;
@ -130,13 +146,83 @@ namespace Ryujinx.Graphics.Vulkan
} }
} }
endRenderPass(); endRenderPass?.Invoke();
PushDataImpl(cbs, dst, dstOffset, data); PushDataImpl(cbs, dst, dstOffset, data);
return true; return true;
} }
private StagingBufferReserved ReserveDataImpl(CommandBufferScoped cbs, int size, int alignment)
{
// Assumes the caller has already determined that there is enough space.
int offset = BitUtils.AlignUp(_freeOffset, alignment);
int padding = offset - _freeOffset;
int capacity = Math.Min(_freeSize, BufferSize - offset);
int reservedLength = size + padding;
if (capacity < size)
{
offset = 0; // Place at start.
reservedLength += capacity;
}
_freeOffset = (_freeOffset + reservedLength) & (BufferSize - 1);
_freeSize -= reservedLength;
Debug.Assert(_freeSize >= 0);
_pendingCopies.Enqueue(new PendingCopy(cbs.GetFence(), reservedLength));
return new StagingBufferReserved(_buffer, offset, size);
}
private int GetContiguousFreeSize(int alignment)
{
int alignedFreeOffset = BitUtils.AlignUp(_freeOffset, alignment);
int padding = alignedFreeOffset - _freeOffset;
// Free regions:
// - Aligned free offset to end (minimum free size - padding)
// - 0 to _freeOffset + freeSize wrapped (only if free area contains 0)
int endOffset = (_freeOffset + _freeSize) & (BufferSize - 1);
return Math.Max(
Math.Min(_freeSize - padding, BufferSize - alignedFreeOffset),
endOffset <= _freeOffset ? Math.Min(_freeSize, endOffset) : 0
);
}
/// <summary>
/// Reserve a range on the staging buffer for the current command buffer and upload data to it.
/// </summary>
/// <param name="cbs">Command buffer to reserve the data on</param>
/// <param name="data">The data to upload</param>
/// <param name="alignment">The required alignment for the buffer offset</param>
/// <returns>The reserved range of the staging buffer</returns>
public unsafe StagingBufferReserved? TryReserveData(CommandBufferScoped cbs, int size, int alignment)
{
if (size > BufferSize)
{
return null;
}
// Temporary reserved data cannot be fragmented.
if (GetContiguousFreeSize(alignment) < size)
{
FreeCompleted();
if (GetContiguousFreeSize(alignment) < size)
{
Logger.Debug?.PrintMsg(LogClass.Gpu, $"Staging buffer out of space to reserve data of size {size}.");
return null;
}
}
return ReserveDataImpl(cbs, size, alignment);
}
private bool WaitFreeCompleted(CommandBufferPool cbp) private bool WaitFreeCompleted(CommandBufferPool cbp)
{ {
if (_pendingCopies.TryPeek(out var pc)) if (_pendingCopies.TryPeek(out var pc))

12
src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs
View file

@ -127,24 +127,24 @@ namespace Ryujinx.Graphics.Vulkan
ReleaseImpl(); ReleaseImpl();
} }
public BufferView GetBufferView(CommandBufferScoped cbs) public BufferView GetBufferView(CommandBufferScoped cbs, bool write)
{ {
_bufferView ??= _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size, ReleaseImpl); _bufferView ??= _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size, ReleaseImpl);
return _bufferView?.Get(cbs, _offset, _size).Value ?? default; return _bufferView?.Get(cbs, _offset, _size, write).Value ?? default;
} }
public BufferView GetBufferView(CommandBufferScoped cbs, Format format) public BufferView GetBufferView(CommandBufferScoped cbs, Format format, bool write)
{ {
var vkFormat = FormatTable.GetFormat(format); var vkFormat = FormatTable.GetFormat(format);
if (vkFormat == VkFormat) if (vkFormat == VkFormat)
{ {
return GetBufferView(cbs); return GetBufferView(cbs, write);
} }
if (_selfManagedViews != null && _selfManagedViews.TryGetValue(format, out var bufferView)) if (_selfManagedViews != null && _selfManagedViews.TryGetValue(format, out var bufferView))
{ {
return bufferView.Get(cbs, _offset, _size).Value; return bufferView.Get(cbs, _offset, _size, write).Value;
} }
bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size, ReleaseImpl); bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size, ReleaseImpl);
@ -154,7 +154,7 @@ namespace Ryujinx.Graphics.Vulkan
(_selfManagedViews ??= new Dictionary<Format, Auto<DisposableBufferView>>()).Add(format, bufferView); (_selfManagedViews ??= new Dictionary<Format, Auto<DisposableBufferView>>()).Add(format, bufferView);
} }
return bufferView?.Get(cbs, _offset, _size).Value ?? default; return bufferView?.Get(cbs, _offset, _size, write).Value ?? default;
} }
} }
} }

13
src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs
View file

@ -4,6 +4,8 @@ namespace Ryujinx.Graphics.Vulkan
{ {
internal struct VertexBufferState internal struct VertexBufferState
{ {
private const int VertexBufferMaxMirrorable = 0x20000;
public static VertexBufferState Null => new(null, 0, 0, 0); public static VertexBufferState Null => new(null, 0, 0, 0);
private readonly int _offset; private readonly int _offset;
@ -88,9 +90,11 @@ namespace Ryujinx.Graphics.Vulkan
if (autoBuffer != null) if (autoBuffer != null)
{ {
var buffer = autoBuffer.Get(cbs, _offset, _size).Value; int offset = _offset;
bool mirrorable = _size <= VertexBufferMaxMirrorable;
var buffer = mirrorable ? autoBuffer.GetMirrorable(cbs, ref offset, _size, out _).Value : autoBuffer.Get(cbs, offset, _size).Value;
updater.BindVertexBuffer(cbs, binding, buffer, (ulong)_offset, (ulong)_size, (ulong)_stride); updater.BindVertexBuffer(cbs, binding, buffer, (ulong)offset, (ulong)_size, (ulong)_stride);
} }
} }
@ -99,6 +103,11 @@ namespace Ryujinx.Graphics.Vulkan
return _buffer == buffer; return _buffer == buffer;
} }
public readonly bool Overlaps(Auto<DisposableBuffer> buffer, int offset, int size)
{
return buffer == _buffer && offset < _offset + _size && offset + size > _offset;
}
public readonly bool Matches(Auto<DisposableBuffer> buffer, int descriptorIndex, int offset, int size, int stride = 0) public readonly bool Matches(Auto<DisposableBuffer> buffer, int descriptorIndex, int offset, int size, int stride = 0)
{ {
return _buffer == buffer && DescriptorIndex == descriptorIndex && _offset == offset && _size == size && _stride == stride; return _buffer == buffer && DescriptorIndex == descriptorIndex && _offset == offset && _size == size && _stride == stride;

12
src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs
View file

@ -293,6 +293,13 @@ namespace Ryujinx.Graphics.Vulkan
ref var properties = ref properties2.Properties; ref var properties = ref properties2.Properties;
ulong minResourceAlignment = Math.Max(
Math.Max(
properties.Limits.MinStorageBufferOffsetAlignment,
properties.Limits.MinUniformBufferOffsetAlignment),
properties.Limits.MinTexelBufferOffsetAlignment
);
SampleCountFlags supportedSampleCounts = SampleCountFlags supportedSampleCounts =
properties.Limits.FramebufferColorSampleCounts & properties.Limits.FramebufferColorSampleCounts &
properties.Limits.FramebufferDepthSampleCounts & properties.Limits.FramebufferDepthSampleCounts &
@ -334,7 +341,8 @@ namespace Ryujinx.Graphics.Vulkan
supportedSampleCounts, supportedSampleCounts,
portabilityFlags, portabilityFlags,
vertexBufferAlignment, vertexBufferAlignment,
properties.Limits.SubTexelPrecisionBits); properties.Limits.SubTexelPrecisionBits,
minResourceAlignment);
IsSharedMemory = MemoryAllocator.IsDeviceMemoryShared(_physicalDevice); IsSharedMemory = MemoryAllocator.IsDeviceMemoryShared(_physicalDevice);
@ -397,7 +405,7 @@ namespace Ryujinx.Graphics.Vulkan
public BufferHandle CreateBuffer(int size, BufferAccess access) public BufferHandle CreateBuffer(int size, BufferAccess access)
{ {
return BufferManager.CreateWithHandle(this, size, access.Convert()); return BufferManager.CreateWithHandle(this, size, access.Convert(), default, access == BufferAccess.Stream);
} }
public BufferHandle CreateBuffer(int size, BufferHandle storageHint) public BufferHandle CreateBuffer(int size, BufferHandle storageHint)