GPU: Use a bitmap to track buffer modified flags. (#3775)
* Initial implementation * Some improvements. * Fix incorrect cast * Performance improvement and improved correctness * Add very fast path when all handles are checked. * Slightly faster * Add comment * De-virtualize region handle All region handles are now bitmap backed. * Remove non-bitmap tracking * Remove unused methods * Add docs, remove unused methods * Address Feedback * Rename file
This commit is contained in:
riperiperi
GitHub
parent
141cf61ff7
commit
3d98e1361b
5 changed files with 696 additions and 74 deletions
199
Ryujinx.Memory/Tracking/BitMap.cs
Normal file
199
Ryujinx.Memory/Tracking/BitMap.cs
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace Ryujinx.Memory.Tracking
|
||||
{
|
||||
/// <summary>
|
||||
/// A bitmap that can check or set large ranges of true/false values at once.
|
||||
/// </summary>
|
||||
struct BitMap
|
||||
{
|
||||
public const int IntSize = 64;
|
||||
|
||||
private const int IntShift = 6;
|
||||
private const int IntMask = IntSize - 1;
|
||||
|
||||
/// <summary>
|
||||
/// Masks representing the bitmap. Least significant bit first, 64-bits per mask.
|
||||
/// </summary>
|
||||
public readonly long[] Masks;
|
||||
|
||||
/// <summary>
|
||||
/// Create a new bitmap.
|
||||
/// </summary>
|
||||
/// <param name="count">The number of bits to reserve</param>
|
||||
public BitMap(int count)
|
||||
{
|
||||
Masks = new long[(count + IntMask) / IntSize];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if any bit in the bitmap is set.
|
||||
/// </summary>
|
||||
/// <returns>True if any bits are set, false otherwise</returns>
|
||||
public bool AnySet()
|
||||
{
|
||||
for (int i = 0; i < Masks.Length; i++)
|
||||
{
|
||||
if (Masks[i] != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if a bit in the bitmap is set.
|
||||
/// </summary>
|
||||
/// <param name="bit">The bit index to check</param>
|
||||
/// <returns>True if the bit is set, false otherwise</returns>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSet(int bit)
|
||||
{
|
||||
int wordIndex = bit >> IntShift;
|
||||
int wordBit = bit & IntMask;
|
||||
|
||||
long wordMask = 1L << wordBit;
|
||||
|
||||
return (Masks[wordIndex] & wordMask) != 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if any bit in a range of bits in the bitmap are set. (inclusive)
|
||||
/// </summary>
|
||||
/// <param name="start">The first bit index to check</param>
|
||||
/// <param name="end">The last bit index to check</param>
|
||||
/// <returns>True if a bit is set, false otherwise</returns>
|
||||
public bool IsSet(int start, int end)
|
||||
{
|
||||
if (start == end)
|
||||
{
|
||||
return IsSet(start);
|
||||
}
|
||||
|
||||
int startIndex = start >> IntShift;
|
||||
int startBit = start & IntMask;
|
||||
long startMask = -1L << startBit;
|
||||
|
||||
int endIndex = end >> IntShift;
|
||||
int endBit = end & IntMask;
|
||||
long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
|
||||
|
||||
if (startIndex == endIndex)
|
||||
{
|
||||
return (Masks[startIndex] & startMask & endMask) != 0;
|
||||
}
|
||||
|
||||
if ((Masks[startIndex] & startMask) != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
for (int i = startIndex + 1; i < endIndex; i++)
|
||||
{
|
||||
if (Masks[i] != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if ((Masks[endIndex] & endMask) != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Set a bit at a specific index to 1.
|
||||
/// </summary>
|
||||
/// <param name="bit">The bit index to set</param>
|
||||
/// <returns>True if the bit is set, false if it was already set</returns>
|
||||
public bool Set(int bit)
|
||||
{
|
||||
int wordIndex = bit >> IntShift;
|
||||
int wordBit = bit & IntMask;
|
||||
|
||||
long wordMask = 1L << wordBit;
|
||||
|
||||
if ((Masks[wordIndex] & wordMask) != 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Masks[wordIndex] |= wordMask;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Set a range of bits in the bitmap to 1.
|
||||
/// </summary>
|
||||
/// <param name="start">The first bit index to set</param>
|
||||
/// <param name="end">The last bit index to set</param>
|
||||
public void SetRange(int start, int end)
|
||||
{
|
||||
if (start == end)
|
||||
{
|
||||
Set(start);
|
||||
return;
|
||||
}
|
||||
|
||||
int startIndex = start >> IntShift;
|
||||
int startBit = start & IntMask;
|
||||
long startMask = -1L << startBit;
|
||||
|
||||
int endIndex = end >> IntShift;
|
||||
int endBit = end & IntMask;
|
||||
long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
|
||||
|
||||
if (startIndex == endIndex)
|
||||
{
|
||||
Masks[startIndex] |= startMask & endMask;
|
||||
}
|
||||
else
|
||||
{
|
||||
Masks[startIndex] |= startMask;
|
||||
|
||||
for (int i = startIndex + 1; i < endIndex; i++)
|
||||
{
|
||||
Masks[i] |= -1;
|
||||
}
|
||||
|
||||
Masks[endIndex] |= endMask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clear a bit at a specific index to 0.
|
||||
/// </summary>
|
||||
/// <param name="bit">The bit index to clear</param>
|
||||
/// <returns>True if the bit was set, false if it was not</returns>
|
||||
public bool Clear(int bit)
|
||||
{
|
||||
int wordIndex = bit >> IntShift;
|
||||
int wordBit = bit & IntMask;
|
||||
|
||||
long wordMask = 1L << wordBit;
|
||||
|
||||
bool wasSet = (Masks[wordIndex] & wordMask) != 0;
|
||||
|
||||
Masks[wordIndex] &= ~wordMask;
|
||||
|
||||
return wasSet;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clear the bitmap entirely, setting all bits to 0.
|
||||
/// </summary>
|
||||
public void Clear()
|
||||
{
|
||||
for (int i = 0; i < Masks.Length; i++)
|
||||
{
|
||||
Masks[i] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
161
Ryujinx.Memory/Tracking/ConcurrentBitmap.cs
Normal file
161
Ryujinx.Memory/Tracking/ConcurrentBitmap.cs
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
using System;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Memory.Tracking
|
||||
{
|
||||
/// <summary>
|
||||
/// A bitmap that can be safely modified from multiple threads.
|
||||
/// </summary>
|
||||
internal class ConcurrentBitmap
|
||||
{
|
||||
public const int IntSize = 64;
|
||||
|
||||
public const int IntShift = 6;
|
||||
public const int IntMask = IntSize - 1;
|
||||
|
||||
/// <summary>
|
||||
/// Masks representing the bitmap. Least significant bit first, 64-bits per mask.
|
||||
/// </summary>
|
||||
public readonly long[] Masks;
|
||||
|
||||
/// <summary>
|
||||
/// Create a new multithreaded bitmap.
|
||||
/// </summary>
|
||||
/// <param name="count">The number of bits to reserve</param>
|
||||
/// <param name="set">Whether the bits should be initially set or not</param>
|
||||
public ConcurrentBitmap(int count, bool set)
|
||||
{
|
||||
Masks = new long[(count + IntMask) / IntSize];
|
||||
|
||||
if (set)
|
||||
{
|
||||
Array.Fill(Masks, -1L);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if any bit in the bitmap is set.
|
||||
/// </summary>
|
||||
/// <returns>True if any bits are set, false otherwise</returns>
|
||||
public bool AnySet()
|
||||
{
|
||||
for (int i = 0; i < Masks.Length; i++)
|
||||
{
|
||||
if (Volatile.Read(ref Masks[i]) != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if a bit in the bitmap is set.
|
||||
/// </summary>
|
||||
/// <param name="bit">The bit index to check</param>
|
||||
/// <returns>True if the bit is set, false otherwise</returns>
|
||||
public bool IsSet(int bit)
|
||||
{
|
||||
int wordIndex = bit >> IntShift;
|
||||
int wordBit = bit & IntMask;
|
||||
|
||||
long wordMask = 1L << wordBit;
|
||||
|
||||
return (Volatile.Read(ref Masks[wordIndex]) & wordMask) != 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if any bit in a range of bits in the bitmap are set. (inclusive)
|
||||
/// </summary>
|
||||
/// <param name="start">The first bit index to check</param>
|
||||
/// <param name="end">The last bit index to check</param>
|
||||
/// <returns>True if a bit is set, false otherwise</returns>
|
||||
public bool IsSet(int start, int end)
|
||||
{
|
||||
if (start == end)
|
||||
{
|
||||
return IsSet(start);
|
||||
}
|
||||
|
||||
int startIndex = start >> IntShift;
|
||||
int startBit = start & IntMask;
|
||||
long startMask = -1L << startBit;
|
||||
|
||||
int endIndex = end >> IntShift;
|
||||
int endBit = end & IntMask;
|
||||
long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
|
||||
|
||||
long startValue = Volatile.Read(ref Masks[startIndex]);
|
||||
|
||||
if (startIndex == endIndex)
|
||||
{
|
||||
return (startValue & startMask & endMask) != 0;
|
||||
}
|
||||
|
||||
if ((startValue & startMask) != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
for (int i = startIndex + 1; i < endIndex; i++)
|
||||
{
|
||||
if (Volatile.Read(ref Masks[i]) != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
long endValue = Volatile.Read(ref Masks[endIndex]);
|
||||
|
||||
if ((endValue & endMask) != 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Set a bit at a specific index to either true or false.
|
||||
/// </summary>
|
||||
/// <param name="bit">The bit index to set</param>
|
||||
/// <param name="value">Whether the bit should be set or not</param>
|
||||
public void Set(int bit, bool value)
|
||||
{
|
||||
int wordIndex = bit >> IntShift;
|
||||
int wordBit = bit & IntMask;
|
||||
|
||||
long wordMask = 1L << wordBit;
|
||||
|
||||
long existing;
|
||||
long newValue;
|
||||
|
||||
do
|
||||
{
|
||||
existing = Volatile.Read(ref Masks[wordIndex]);
|
||||
|
||||
if (value)
|
||||
{
|
||||
newValue = existing | wordMask;
|
||||
}
|
||||
else
|
||||
{
|
||||
newValue = existing & ~wordMask;
|
||||
}
|
||||
}
|
||||
while (Interlocked.CompareExchange(ref Masks[wordIndex], newValue, existing) != existing);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clear the bitmap entirely, setting all bits to 0.
|
||||
/// </summary>
|
||||
public void Clear()
|
||||
{
|
||||
for (int i = 0; i < Masks.Length; i++)
|
||||
{
|
||||
Volatile.Write(ref Masks[i], 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -176,6 +176,26 @@ namespace Ryujinx.Memory.Tracking
|
|||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with.
|
||||
/// </summary>
|
||||
/// <param name="address">CPU virtual address of the region</param>
|
||||
/// <param name="size">Size of the region</param>
|
||||
/// <param name="bitmap">The bitmap owning the dirty flag for this handle</param>
|
||||
/// <param name="bit">The bit of this handle within the dirty flag</param>
|
||||
/// <returns>The memory tracking handle</returns>
|
||||
internal RegionHandle BeginTrackingBitmap(ulong address, ulong size, ConcurrentBitmap bitmap, int bit)
|
||||
{
|
||||
(address, size) = PageAlign(address, size);
|
||||
|
||||
lock (TrackingLock)
|
||||
{
|
||||
RegionHandle handle = new RegionHandle(this, address, size, bitmap, bit, _memoryManager.IsRangeMapped(address, size));
|
||||
|
||||
return handle;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Signal that a virtual memory event happened at the given location (one byte).
|
||||
/// </summary>
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Memory.Tracking
|
||||
{
|
||||
/// <summary>
|
||||
/// A region handle that tracks a large region using many smaller handles, to provide
|
||||
/// granular tracking that can be used to track partial updates.
|
||||
/// granular tracking that can be used to track partial updates. Backed by a bitmap
|
||||
/// to improve performance when scanning large regions.
|
||||
/// </summary>
|
||||
public class MultiRegionHandle : IMultiRegionHandle
|
||||
{
|
||||
|
|
@ -17,6 +21,12 @@ namespace Ryujinx.Memory.Tracking
|
|||
private readonly ulong Granularity;
|
||||
private readonly ulong Size;
|
||||
|
||||
private ConcurrentBitmap _dirtyBitmap;
|
||||
|
||||
private int _sequenceNumber;
|
||||
private BitMap _sequenceNumberBitmap;
|
||||
private int _uncheckedHandles;
|
||||
|
||||
public bool Dirty { get; private set; } = true;
|
||||
|
||||
internal MultiRegionHandle(MemoryTracking tracking, ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity)
|
||||
|
|
@ -24,6 +34,9 @@ namespace Ryujinx.Memory.Tracking
|
|||
_handles = new RegionHandle[size / granularity];
|
||||
Granularity = granularity;
|
||||
|
||||
_dirtyBitmap = new ConcurrentBitmap(_handles.Length, true);
|
||||
_sequenceNumberBitmap = new BitMap(_handles.Length);
|
||||
|
||||
int i = 0;
|
||||
|
||||
if (handles != null)
|
||||
|
|
@ -40,37 +53,43 @@ namespace Ryujinx.Memory.Tracking
|
|||
// Fill any gap left before this handle.
|
||||
while (i < startIndex)
|
||||
{
|
||||
RegionHandle fillHandle = tracking.BeginTracking(address + (ulong)i * granularity, granularity);
|
||||
RegionHandle fillHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
|
||||
fillHandle.Parent = this;
|
||||
_handles[i++] = fillHandle;
|
||||
}
|
||||
|
||||
if (handle.Size == granularity)
|
||||
lock (tracking.TrackingLock)
|
||||
{
|
||||
handle.Parent = this;
|
||||
_handles[i++] = handle;
|
||||
}
|
||||
else
|
||||
{
|
||||
int endIndex = (int)((handle.EndAddress - address) / granularity);
|
||||
|
||||
while (i < endIndex)
|
||||
if (handle is RegionHandle bitHandle && handle.Size == granularity)
|
||||
{
|
||||
RegionHandle splitHandle = tracking.BeginTracking(address + (ulong)i * granularity, granularity);
|
||||
splitHandle.Parent = this;
|
||||
handle.Parent = this;
|
||||
|
||||
splitHandle.Reprotect(handle.Dirty);
|
||||
bitHandle.ReplaceBitmap(_dirtyBitmap, i);
|
||||
|
||||
RegionSignal signal = handle.PreAction;
|
||||
if (signal != null)
|
||||
_handles[i++] = bitHandle;
|
||||
}
|
||||
else
|
||||
{
|
||||
int endIndex = (int)((handle.EndAddress - address) / granularity);
|
||||
|
||||
while (i < endIndex)
|
||||
{
|
||||
splitHandle.RegisterAction(signal);
|
||||
RegionHandle splitHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
|
||||
splitHandle.Parent = this;
|
||||
|
||||
splitHandle.Reprotect(handle.Dirty);
|
||||
|
||||
RegionSignal signal = handle.PreAction;
|
||||
if (signal != null)
|
||||
{
|
||||
splitHandle.RegisterAction(signal);
|
||||
}
|
||||
|
||||
_handles[i++] = splitHandle;
|
||||
}
|
||||
|
||||
_handles[i++] = splitHandle;
|
||||
handle.Dispose();
|
||||
}
|
||||
|
||||
handle.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -78,15 +97,27 @@ namespace Ryujinx.Memory.Tracking
|
|||
// Fill any remaining space with new handles.
|
||||
while (i < _handles.Length)
|
||||
{
|
||||
RegionHandle handle = tracking.BeginTracking(address + (ulong)i * granularity, granularity);
|
||||
RegionHandle handle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
|
||||
handle.Parent = this;
|
||||
_handles[i++] = handle;
|
||||
}
|
||||
|
||||
_uncheckedHandles = _handles.Length;
|
||||
|
||||
Address = address;
|
||||
Size = size;
|
||||
}
|
||||
|
||||
public void SignalWrite()
|
||||
{
|
||||
Dirty = true;
|
||||
}
|
||||
|
||||
public IEnumerable<RegionHandle> GetHandles()
|
||||
{
|
||||
return _handles;
|
||||
}
|
||||
|
||||
public void ForceDirty(ulong address, ulong size)
|
||||
{
|
||||
Dirty = true;
|
||||
|
|
@ -96,21 +127,15 @@ namespace Ryujinx.Memory.Tracking
|
|||
|
||||
for (int i = startHandle; i <= lastHandle; i++)
|
||||
{
|
||||
_handles[i].SequenceNumber--;
|
||||
if (_sequenceNumberBitmap.Clear(i))
|
||||
{
|
||||
_uncheckedHandles++;
|
||||
}
|
||||
|
||||
_handles[i].ForceDirty();
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerable<RegionHandle> GetHandles()
|
||||
{
|
||||
return _handles;
|
||||
}
|
||||
|
||||
public void SignalWrite()
|
||||
{
|
||||
Dirty = true;
|
||||
}
|
||||
|
||||
public void QueryModified(Action<ulong, ulong> modifiedAction)
|
||||
{
|
||||
if (!Dirty)
|
||||
|
|
@ -123,33 +148,95 @@ namespace Ryujinx.Memory.Tracking
|
|||
QueryModified(Address, Size, modifiedAction);
|
||||
}
|
||||
|
||||
public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void ParseDirtyBits(long dirtyBits, ref int baseBit, ref int prevHandle, ref ulong rgStart, ref ulong rgSize, Action<ulong, ulong> modifiedAction)
|
||||
{
|
||||
int startHandle = (int)((address - Address) / Granularity);
|
||||
int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
|
||||
|
||||
ulong rgStart = _handles[startHandle].Address;
|
||||
ulong rgSize = 0;
|
||||
|
||||
for (int i = startHandle; i <= lastHandle; i++)
|
||||
while (dirtyBits != 0)
|
||||
{
|
||||
RegionHandle handle = _handles[i];
|
||||
int bit = BitOperations.TrailingZeroCount(dirtyBits);
|
||||
|
||||
dirtyBits &= ~(1L << bit);
|
||||
|
||||
int handleIndex = baseBit + bit;
|
||||
|
||||
RegionHandle handle = _handles[handleIndex];
|
||||
|
||||
if (handleIndex != prevHandle + 1)
|
||||
{
|
||||
// Submit handles scanned until the gap as dirty
|
||||
if (rgSize != 0)
|
||||
{
|
||||
modifiedAction(rgStart, rgSize);
|
||||
rgSize = 0;
|
||||
}
|
||||
rgStart = handle.Address;
|
||||
}
|
||||
|
||||
if (handle.Dirty)
|
||||
{
|
||||
rgSize += handle.Size;
|
||||
handle.Reprotect();
|
||||
}
|
||||
else
|
||||
|
||||
prevHandle = handleIndex;
|
||||
}
|
||||
|
||||
baseBit += ConcurrentBitmap.IntSize;
|
||||
}
|
||||
|
||||
public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction)
|
||||
{
|
||||
int startHandle = (int)((address - Address) / Granularity);
|
||||
int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
|
||||
|
||||
ulong rgStart = _handles[startHandle].Address;
|
||||
|
||||
if (startHandle == lastHandle)
|
||||
{
|
||||
RegionHandle handle = _handles[startHandle];
|
||||
|
||||
if (handle.Dirty)
|
||||
{
|
||||
// Submit the region scanned so far as dirty
|
||||
if (rgSize != 0)
|
||||
{
|
||||
modifiedAction(rgStart, rgSize);
|
||||
rgSize = 0;
|
||||
}
|
||||
rgStart = handle.EndAddress;
|
||||
handle.Reprotect();
|
||||
modifiedAction(rgStart, handle.Size);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
ulong rgSize = 0;
|
||||
|
||||
long[] masks = _dirtyBitmap.Masks;
|
||||
|
||||
int startIndex = startHandle >> ConcurrentBitmap.IntShift;
|
||||
int startBit = startHandle & ConcurrentBitmap.IntMask;
|
||||
long startMask = -1L << startBit;
|
||||
|
||||
int endIndex = lastHandle >> ConcurrentBitmap.IntShift;
|
||||
int endBit = lastHandle & ConcurrentBitmap.IntMask;
|
||||
long endMask = (long)(ulong.MaxValue >> (ConcurrentBitmap.IntMask - endBit));
|
||||
|
||||
long startValue = Volatile.Read(ref masks[startIndex]);
|
||||
|
||||
int baseBit = startIndex << ConcurrentBitmap.IntShift;
|
||||
int prevHandle = startHandle - 1;
|
||||
|
||||
if (startIndex == endIndex)
|
||||
{
|
||||
ParseDirtyBits(startValue & startMask & endMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
|
||||
}
|
||||
else
|
||||
{
|
||||
ParseDirtyBits(startValue & startMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
|
||||
|
||||
for (int i = startIndex + 1; i < endIndex; i++)
|
||||
{
|
||||
ParseDirtyBits(Volatile.Read(ref masks[i]), ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
|
||||
}
|
||||
|
||||
long endValue = Volatile.Read(ref masks[endIndex]);
|
||||
|
||||
ParseDirtyBits(endValue & endMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
|
||||
}
|
||||
|
||||
if (rgSize != 0)
|
||||
|
|
@ -158,35 +245,120 @@ namespace Ryujinx.Memory.Tracking
|
|||
}
|
||||
}
|
||||
|
||||
public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction, int sequenceNumber)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void ParseDirtyBits(long dirtyBits, long mask, int index, long[] seqMasks, ref int baseBit, ref int prevHandle, ref ulong rgStart, ref ulong rgSize, Action<ulong, ulong> modifiedAction)
|
||||
{
|
||||
int startHandle = (int)((address - Address) / Granularity);
|
||||
int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
|
||||
long seqMask = mask & ~seqMasks[index];
|
||||
dirtyBits &= seqMask;
|
||||
|
||||
ulong rgStart = _handles[startHandle].Address;
|
||||
ulong rgSize = 0;
|
||||
|
||||
for (int i = startHandle; i <= lastHandle; i++)
|
||||
while (dirtyBits != 0)
|
||||
{
|
||||
RegionHandle handle = _handles[i];
|
||||
int bit = BitOperations.TrailingZeroCount(dirtyBits);
|
||||
|
||||
if (sequenceNumber != handle.SequenceNumber && handle.DirtyOrVolatile())
|
||||
dirtyBits &= ~(1L << bit);
|
||||
|
||||
int handleIndex = baseBit + bit;
|
||||
|
||||
RegionHandle handle = _handles[handleIndex];
|
||||
|
||||
if (handleIndex != prevHandle + 1)
|
||||
{
|
||||
rgSize += handle.Size;
|
||||
handle.Reprotect();
|
||||
}
|
||||
else
|
||||
{
|
||||
// Submit the region scanned so far as dirty
|
||||
// Submit handles scanned until the gap as dirty
|
||||
if (rgSize != 0)
|
||||
{
|
||||
modifiedAction(rgStart, rgSize);
|
||||
rgSize = 0;
|
||||
}
|
||||
rgStart = handle.EndAddress;
|
||||
rgStart = handle.Address;
|
||||
}
|
||||
|
||||
handle.SequenceNumber = sequenceNumber;
|
||||
rgSize += handle.Size;
|
||||
handle.Reprotect();
|
||||
|
||||
prevHandle = handleIndex;
|
||||
}
|
||||
|
||||
seqMasks[index] |= mask;
|
||||
_uncheckedHandles -= BitOperations.PopCount((ulong)seqMask);
|
||||
|
||||
baseBit += ConcurrentBitmap.IntSize;
|
||||
}
|
||||
|
||||
public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction, int sequenceNumber)
|
||||
{
|
||||
int startHandle = (int)((address - Address) / Granularity);
|
||||
int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
|
||||
|
||||
ulong rgStart = Address + (ulong)startHandle * Granularity;
|
||||
|
||||
if (sequenceNumber != _sequenceNumber)
|
||||
{
|
||||
if (_uncheckedHandles != _handles.Length)
|
||||
{
|
||||
_sequenceNumberBitmap.Clear();
|
||||
_uncheckedHandles = _handles.Length;
|
||||
}
|
||||
|
||||
_sequenceNumber = sequenceNumber;
|
||||
}
|
||||
|
||||
if (startHandle == lastHandle)
|
||||
{
|
||||
var handle = _handles[startHandle];
|
||||
if (_sequenceNumberBitmap.Set(startHandle))
|
||||
{
|
||||
_uncheckedHandles--;
|
||||
|
||||
if (handle.DirtyOrVolatile())
|
||||
{
|
||||
handle.Reprotect();
|
||||
|
||||
modifiedAction(rgStart, handle.Size);
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (_uncheckedHandles == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ulong rgSize = 0;
|
||||
|
||||
long[] seqMasks = _sequenceNumberBitmap.Masks;
|
||||
long[] masks = _dirtyBitmap.Masks;
|
||||
|
||||
int startIndex = startHandle >> ConcurrentBitmap.IntShift;
|
||||
int startBit = startHandle & ConcurrentBitmap.IntMask;
|
||||
long startMask = -1L << startBit;
|
||||
|
||||
int endIndex = lastHandle >> ConcurrentBitmap.IntShift;
|
||||
int endBit = lastHandle & ConcurrentBitmap.IntMask;
|
||||
long endMask = (long)(ulong.MaxValue >> (ConcurrentBitmap.IntMask - endBit));
|
||||
|
||||
long startValue = Volatile.Read(ref masks[startIndex]);
|
||||
|
||||
int baseBit = startIndex << ConcurrentBitmap.IntShift;
|
||||
int prevHandle = startHandle - 1;
|
||||
|
||||
if (startIndex == endIndex)
|
||||
{
|
||||
ParseDirtyBits(startValue, startMask & endMask, startIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
|
||||
}
|
||||
else
|
||||
{
|
||||
ParseDirtyBits(startValue, startMask, startIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
|
||||
|
||||
for (int i = startIndex + 1; i < endIndex; i++)
|
||||
{
|
||||
ParseDirtyBits(Volatile.Read(ref masks[i]), -1L, i, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
|
||||
}
|
||||
|
||||
long endValue = Volatile.Read(ref masks[endIndex]);
|
||||
|
||||
ParseDirtyBits(endValue, endMask, endIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
|
||||
}
|
||||
|
||||
if (rgSize != 0)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
using Ryujinx.Memory.Range;
|
||||
using System;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
|
|
@ -10,7 +9,7 @@ namespace Ryujinx.Memory.Tracking
|
|||
/// A tracking handle for a given region of virtual memory. The Dirty flag is updated whenever any changes are made,
|
||||
/// and an action can be performed when the region is read to or written from.
|
||||
/// </summary>
|
||||
public class RegionHandle : IRegionHandle, IRange
|
||||
public class RegionHandle : IRegionHandle
|
||||
{
|
||||
/// <summary>
|
||||
/// If more than this number of checks have been performed on a dirty flag since its last reprotect,
|
||||
|
|
@ -23,7 +22,20 @@ namespace Ryujinx.Memory.Tracking
|
|||
/// </summary>
|
||||
private static int VolatileThreshold = 5;
|
||||
|
||||
public bool Dirty { get; private set; }
|
||||
public bool Dirty
|
||||
{
|
||||
get
|
||||
{
|
||||
return Bitmap.IsSet(DirtyBit);
|
||||
}
|
||||
protected set
|
||||
{
|
||||
Bitmap.Set(DirtyBit, value);
|
||||
}
|
||||
}
|
||||
|
||||
internal int SequenceNumber { get; set; }
|
||||
|
||||
public bool Unmapped { get; private set; }
|
||||
|
||||
public ulong Address { get; }
|
||||
|
|
@ -31,7 +43,6 @@ namespace Ryujinx.Memory.Tracking
|
|||
public ulong EndAddress { get; }
|
||||
|
||||
internal IMultiRegionHandle Parent { get; set; }
|
||||
internal int SequenceNumber { get; set; }
|
||||
|
||||
private event Action _onDirty;
|
||||
|
||||
|
|
@ -68,17 +79,26 @@ namespace Ryujinx.Memory.Tracking
|
|||
|
||||
internal RegionSignal PreAction => _preAction;
|
||||
|
||||
internal ConcurrentBitmap Bitmap;
|
||||
internal int DirtyBit;
|
||||
|
||||
/// <summary>
|
||||
/// Create a new region handle. The handle is registered with the given tracking object,
|
||||
/// Create a new bitmap backed region handle. The handle is registered with the given tracking object,
|
||||
/// and will be notified of any changes to the specified region.
|
||||
/// </summary>
|
||||
/// <param name="tracking">Tracking object for the target memory block</param>
|
||||
/// <param name="address">Virtual address of the region to track</param>
|
||||
/// <param name="size">Size of the region to track</param>
|
||||
/// <param name="bitmap">The bitmap the dirty flag for this handle is stored in</param>
|
||||
/// <param name="bit">The bit index representing the dirty flag for this handle</param>
|
||||
/// <param name="mapped">True if the region handle starts mapped</param>
|
||||
internal RegionHandle(MemoryTracking tracking, ulong address, ulong size, bool mapped = true)
|
||||
internal RegionHandle(MemoryTracking tracking, ulong address, ulong size, ConcurrentBitmap bitmap, int bit, bool mapped = true)
|
||||
{
|
||||
Bitmap = bitmap;
|
||||
DirtyBit = bit;
|
||||
|
||||
Dirty = mapped;
|
||||
|
||||
Unmapped = !mapped;
|
||||
Address = address;
|
||||
Size = size;
|
||||
|
|
@ -92,6 +112,54 @@ namespace Ryujinx.Memory.Tracking
|
|||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a new region handle. The handle is registered with the given tracking object,
|
||||
/// and will be notified of any changes to the specified region.
|
||||
/// </summary>
|
||||
/// <param name="tracking">Tracking object for the target memory block</param>
|
||||
/// <param name="address">Virtual address of the region to track</param>
|
||||
/// <param name="size">Size of the region to track</param>
|
||||
/// <param name="mapped">True if the region handle starts mapped</param>
|
||||
internal RegionHandle(MemoryTracking tracking, ulong address, ulong size, bool mapped = true)
|
||||
{
|
||||
Bitmap = new ConcurrentBitmap(1, mapped);
|
||||
|
||||
Unmapped = !mapped;
|
||||
Address = address;
|
||||
Size = size;
|
||||
EndAddress = address + size;
|
||||
|
||||
_tracking = tracking;
|
||||
_regions = tracking.GetVirtualRegionsForHandle(address, size);
|
||||
foreach (var region in _regions)
|
||||
{
|
||||
region.Handles.Add(this);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replace the bitmap and bit index used to track dirty state.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The tracking lock should be held when this is called, to ensure neither bitmap is modified.
|
||||
/// </remarks>
|
||||
/// <param name="bitmap">The bitmap the dirty flag for this handle is stored in</param>
|
||||
/// <param name="bit">The bit index representing the dirty flag for this handle</param>
|
||||
internal void ReplaceBitmap(ConcurrentBitmap bitmap, int bit)
|
||||
{
|
||||
// Assumes the tracking lock is held, so nothing else can signal right now.
|
||||
|
||||
var oldBitmap = Bitmap;
|
||||
var oldBit = DirtyBit;
|
||||
|
||||
bitmap.Set(bit, Dirty);
|
||||
|
||||
Bitmap = bitmap;
|
||||
DirtyBit = bit;
|
||||
|
||||
Dirty |= oldBitmap.IsSet(oldBit);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clear the volatile state of this handle.
|
||||
/// </summary>
|
||||
|
|
@ -108,7 +176,7 @@ namespace Ryujinx.Memory.Tracking
|
|||
public bool DirtyOrVolatile()
|
||||
{
|
||||
_checkCount++;
|
||||
return Dirty || _volatile;
|
||||
return _volatile || Dirty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -195,6 +263,8 @@ namespace Ryujinx.Memory.Tracking
|
|||
/// </summary>
|
||||
public void ForceDirty()
|
||||
{
|
||||
_checkCount++;
|
||||
|
||||
Dirty = true;
|
||||
}
|
||||
|
||||
|
|
|
|||
Reference in a new issue