GPU: Use a bitmap to track buffer modified flags. (#3775)

* Initial implementation

* Some improvements.

* Fix incorrect cast

* Performance improvement and improved correctness

* Add very fast path when all handles are checked.

* Slightly faster

* Add comment

* De-virtualize region handle

All region handles are now bitmap backed.

* Remove non-bitmap tracking

* Remove unused methods

* Add docs, remove unused methods

* Address Feedback

* Rename file

Ryujinx.Memory/Tracking/BitMap.cs

@@ -0,0 +1,199 @@
+using System.Runtime.CompilerServices;
+
+namespace Ryujinx.Memory.Tracking
+{
+    /// <summary>
+    /// A bitmap that can check or set large ranges of true/false values at once.
+    /// </summary>
+    struct BitMap
+    {
+        public const int IntSize = 64;
+
+        private const int IntShift = 6;
+        private const int IntMask = IntSize - 1;
+
+        /// <summary>
+        /// Masks representing the bitmap. Least significant bit first, 64-bits per mask.
+        /// </summary>
+        public readonly long[] Masks;
+
+        /// <summary>
+        /// Create a new bitmap.
+        /// </summary>
+        /// <param name="count">The number of bits to reserve</param>
+        public BitMap(int count)
+        {
+            Masks = new long[(count + IntMask) / IntSize];
+        }
+
+        /// <summary>
+        /// Check if any bit in the bitmap is set.
+        /// </summary>
+        /// <returns>True if any bits are set, false otherwise</returns>
+        public bool AnySet()
+        {
+            for (int i = 0; i < Masks.Length; i++)
+            {
+                if (Masks[i] != 0)
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Check if a bit in the bitmap is set.
+        /// </summary>
+        /// <param name="bit">The bit index to check</param>
+        /// <returns>True if the bit is set, false otherwise</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool IsSet(int bit)
+        {
+            int wordIndex = bit >> IntShift;
+            int wordBit = bit & IntMask;
+
+            long wordMask = 1L << wordBit;
+
+            return (Masks[wordIndex] & wordMask) != 0;
+        }
+
+        /// <summary>
+        /// Check if any bit in a range of bits in the bitmap are set. (inclusive)
+        /// </summary>
+        /// <param name="start">The first bit index to check</param>
+        /// <param name="end">The last bit index to check</param>
+        /// <returns>True if a bit is set, false otherwise</returns>
+        public bool IsSet(int start, int end)
+        {
+            if (start == end)
+            {
+                return IsSet(start);
+            }
+
+            int startIndex = start >> IntShift;
+            int startBit = start & IntMask;
+            long startMask = -1L << startBit;
+
+            int endIndex = end >> IntShift;
+            int endBit = end & IntMask;
+            long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
+
+            if (startIndex == endIndex)
+            {
+                return (Masks[startIndex] & startMask & endMask) != 0;
+            }
+
+            if ((Masks[startIndex] & startMask) != 0)
+            {
+                return true;
+            }
+
+            for (int i = startIndex + 1; i < endIndex; i++)
+            {
+                if (Masks[i] != 0)
+                {
+                    return true;
+                }
+            }
+
+            if ((Masks[endIndex] & endMask) != 0)
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Set a bit at a specific index to 1.
+        /// </summary>
+        /// <param name="bit">The bit index to set</param>
+        /// <returns>True if the bit is set, false if it was already set</returns>
+        public bool Set(int bit)
+        {
+            int wordIndex = bit >> IntShift;
+            int wordBit = bit & IntMask;
+
+            long wordMask = 1L << wordBit;
+
+            if ((Masks[wordIndex] & wordMask) != 0)
+            {
+                return false;
+            }
+
+            Masks[wordIndex] |= wordMask;
+
+            return true;
+        }
+
+        /// <summary>
+        /// Set a range of bits in the bitmap to 1.
+        /// </summary>
+        /// <param name="start">The first bit index to set</param>
+        /// <param name="end">The last bit index to set</param>
+        public void SetRange(int start, int end)
+        {
+            if (start == end)
+            {
+                Set(start);
+                return;
+            }
+
+            int startIndex = start >> IntShift;
+            int startBit = start & IntMask;
+            long startMask = -1L << startBit;
+
+            int endIndex = end >> IntShift;
+            int endBit = end & IntMask;
+            long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
+
+            if (startIndex == endIndex)
+            {
+                Masks[startIndex] |= startMask & endMask;
+            }
+            else
+            {
+                Masks[startIndex] |= startMask;
+
+                for (int i = startIndex + 1; i < endIndex; i++)
+                {
+                    Masks[i] |= -1;
+                }
+
+                Masks[endIndex] |= endMask;
+            }
+        }
+
+        /// <summary>
+        /// Clear a bit at a specific index to 0.
+        /// </summary>
+        /// <param name="bit">The bit index to clear</param>
+        /// <returns>True if the bit was set, false if it was not</returns>
+        public bool Clear(int bit)
+        {
+            int wordIndex = bit >> IntShift;
+            int wordBit = bit & IntMask;
+
+            long wordMask = 1L << wordBit;
+
+            bool wasSet = (Masks[wordIndex] & wordMask) != 0;
+
+            Masks[wordIndex] &= ~wordMask;
+
+            return wasSet;
+        }
+
+        /// <summary>
+        /// Clear the bitmap entirely, setting all bits to 0.
+        /// </summary>
+        public void Clear()
+        {
+            for (int i = 0; i < Masks.Length; i++)
+            {
+                Masks[i] = 0;
+            }
+        }
+    }
+}

Ryujinx.Memory/Tracking/ConcurrentBitmap.cs

@@ -0,0 +1,161 @@
+using System;
+using System.Threading;
+
+namespace Ryujinx.Memory.Tracking
+{
+    /// <summary>
+    /// A bitmap that can be safely modified from multiple threads.
+    /// </summary>
+    internal class ConcurrentBitmap
+    {
+        public const int IntSize = 64;
+
+        public const int IntShift = 6;
+        public const int IntMask = IntSize - 1;
+
+        /// <summary>
+        /// Masks representing the bitmap. Least significant bit first, 64-bits per mask.
+        /// </summary>
+        public readonly long[] Masks;
+
+        /// <summary>
+        /// Create a new multithreaded bitmap.
+        /// </summary>
+        /// <param name="count">The number of bits to reserve</param>
+        /// <param name="set">Whether the bits should be initially set or not</param>
+        public ConcurrentBitmap(int count, bool set)
+        {
+            Masks = new long[(count + IntMask) / IntSize];
+
+            if (set)
+            {
+                Array.Fill(Masks, -1L);
+            }
+        }
+
+        /// <summary>
+        /// Check if any bit in the bitmap is set.
+        /// </summary>
+        /// <returns>True if any bits are set, false otherwise</returns>
+        public bool AnySet()
+        {
+            for (int i = 0; i < Masks.Length; i++)
+            {
+                if (Volatile.Read(ref Masks[i]) != 0)
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Check if a bit in the bitmap is set.
+        /// </summary>
+        /// <param name="bit">The bit index to check</param>
+        /// <returns>True if the bit is set, false otherwise</returns>
+        public bool IsSet(int bit)
+        {
+            int wordIndex = bit >> IntShift;
+            int wordBit = bit & IntMask;
+
+            long wordMask = 1L << wordBit;
+
+            return (Volatile.Read(ref Masks[wordIndex]) & wordMask) != 0;
+        }
+
+        /// <summary>
+        /// Check if any bit in a range of bits in the bitmap are set. (inclusive)
+        /// </summary>
+        /// <param name="start">The first bit index to check</param>
+        /// <param name="end">The last bit index to check</param>
+        /// <returns>True if a bit is set, false otherwise</returns>
+        public bool IsSet(int start, int end)
+        {
+            if (start == end)
+            {
+                return IsSet(start);
+            }
+
+            int startIndex = start >> IntShift;
+            int startBit = start & IntMask;
+            long startMask = -1L << startBit;
+
+            int endIndex = end >> IntShift;
+            int endBit = end & IntMask;
+            long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
+
+            long startValue = Volatile.Read(ref Masks[startIndex]);
+
+            if (startIndex == endIndex)
+            {
+                return (startValue & startMask & endMask) != 0;
+            }
+
+            if ((startValue & startMask) != 0)
+            {
+                return true;
+            }
+
+            for (int i = startIndex + 1; i < endIndex; i++)
+            {
+                if (Volatile.Read(ref Masks[i]) != 0)
+                {
+                    return true;
+                }
+            }
+
+            long endValue = Volatile.Read(ref Masks[endIndex]);
+
+            if ((endValue & endMask) != 0)
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Set a bit at a specific index to either true or false.
+        /// </summary>
+        /// <param name="bit">The bit index to set</param>
+        /// <param name="value">Whether the bit should be set or not</param>
+        public void Set(int bit, bool value)
+        {
+            int wordIndex = bit >> IntShift;
+            int wordBit = bit & IntMask;
+
+            long wordMask = 1L << wordBit;
+
+            long existing;
+            long newValue;
+
+            do
+            {
+                existing = Volatile.Read(ref Masks[wordIndex]);
+
+                if (value)
+                {
+                    newValue = existing | wordMask;
+                }
+                else
+                {
+                    newValue = existing & ~wordMask;
+                }
+            }
+            while (Interlocked.CompareExchange(ref Masks[wordIndex], newValue, existing) != existing);
+        }
+
+        /// <summary>
+        /// Clear the bitmap entirely, setting all bits to 0.
+        /// </summary>
+        public void Clear()
+        {
+            for (int i = 0; i < Masks.Length; i++)
+            {
+                Volatile.Write(ref Masks[i], 0);
+            }
+        }
+    }
+}

Ryujinx.Memory/Tracking/MemoryTracking.cs

@@ -176,6 +176,26 @@ namespace Ryujinx.Memory.Tracking
             }
         }
 
+        /// <summary>
+        /// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with.
+        /// </summary>
+        /// <param name="address">CPU virtual address of the region</param>
+        /// <param name="size">Size of the region</param>
+        /// <param name="bitmap">The bitmap owning the dirty flag for this handle</param>
+        /// <param name="bit">The bit of this handle within the dirty flag</param>
+        /// <returns>The memory tracking handle</returns>
+        internal RegionHandle BeginTrackingBitmap(ulong address, ulong size, ConcurrentBitmap bitmap, int bit)
+        {
+            (address, size) = PageAlign(address, size);
+
+            lock (TrackingLock)
+            {
+                RegionHandle handle = new RegionHandle(this, address, size, bitmap, bit, _memoryManager.IsRangeMapped(address, size));
+
+                return handle;
+            }
+        }
+
         /// <summary>
         /// Signal that a virtual memory event happened at the given location (one byte).
         /// </summary>

Ryujinx.Memory/Tracking/MultiRegionHandle.cs

@@ -1,11 +1,15 @@
 using System;
 using System.Collections.Generic;
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Threading;
 
 namespace Ryujinx.Memory.Tracking
 {
     /// <summary>
     /// A region handle that tracks a large region using many smaller handles, to provide
-    /// granular tracking that can be used to track partial updates.
+    /// granular tracking that can be used to track partial updates. Backed by a bitmap
+    /// to improve performance when scanning large regions.
     /// </summary>
     public class MultiRegionHandle : IMultiRegionHandle
     {
@@ -17,6 +21,12 @@ namespace Ryujinx.Memory.Tracking
         private readonly ulong Granularity;
         private readonly ulong Size;
 
+        private ConcurrentBitmap _dirtyBitmap;
+
+        private int _sequenceNumber;
+        private BitMap _sequenceNumberBitmap;
+        private int _uncheckedHandles;
+
         public bool Dirty { get; private set; } = true;
 
         internal MultiRegionHandle(MemoryTracking tracking, ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity)
@@ -24,6 +34,9 @@ namespace Ryujinx.Memory.Tracking
             _handles = new RegionHandle[size / granularity];
             Granularity = granularity;
 
+            _dirtyBitmap = new ConcurrentBitmap(_handles.Length, true);
+            _sequenceNumberBitmap = new BitMap(_handles.Length);
+
             int i = 0;
 
             if (handles != null)
@@ -40,37 +53,43 @@ namespace Ryujinx.Memory.Tracking
                     // Fill any gap left before this handle.
                     while (i < startIndex)
                     {
-                        RegionHandle fillHandle = tracking.BeginTracking(address + (ulong)i * granularity, granularity);
+                        RegionHandle fillHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
                         fillHandle.Parent = this;
                         _handles[i++] = fillHandle;
                     }
 
-                    if (handle.Size == granularity)
+                    lock (tracking.TrackingLock)
                     {
-                        handle.Parent = this;
-                        _handles[i++] = handle;
-                    }
-                    else
-                    {
-                        int endIndex = (int)((handle.EndAddress - address) / granularity);
-
-                        while (i < endIndex)
+                        if (handle is RegionHandle bitHandle && handle.Size == granularity)
                         {
-                            RegionHandle splitHandle = tracking.BeginTracking(address + (ulong)i * granularity, granularity);
-                            splitHandle.Parent = this;
+                            handle.Parent = this;
 
-                            splitHandle.Reprotect(handle.Dirty);
+                            bitHandle.ReplaceBitmap(_dirtyBitmap, i);
 
-                            RegionSignal signal = handle.PreAction;
-                            if (signal != null)
+                            _handles[i++] = bitHandle;
+                        }
+                        else
+                        {
+                            int endIndex = (int)((handle.EndAddress - address) / granularity);
+
+                            while (i < endIndex)
                             {
-                                splitHandle.RegisterAction(signal);
+                                RegionHandle splitHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
+                                splitHandle.Parent = this;
+
+                                splitHandle.Reprotect(handle.Dirty);
+
+                                RegionSignal signal = handle.PreAction;
+                                if (signal != null)
+                                {
+                                    splitHandle.RegisterAction(signal);
+                                }
+
+                                _handles[i++] = splitHandle;
                             }
 
-                            _handles[i++] = splitHandle;
+                            handle.Dispose();
                         }
-
-                        handle.Dispose();
                     }
                 }
             }
@@ -78,15 +97,27 @@ namespace Ryujinx.Memory.Tracking
             // Fill any remaining space with new handles.
             while (i < _handles.Length)
             {
-                RegionHandle handle = tracking.BeginTracking(address + (ulong)i * granularity, granularity);
+                RegionHandle handle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
                 handle.Parent = this;
                 _handles[i++] = handle;
             }
 
+            _uncheckedHandles = _handles.Length;
+
             Address = address;
             Size = size;
         }
 
+        public void SignalWrite()
+        {
+            Dirty = true;
+        }
+
+        public IEnumerable<RegionHandle> GetHandles()
+        {
+            return _handles;
+        }
+
         public void ForceDirty(ulong address, ulong size)
         {
             Dirty = true;
@@ -96,21 +127,15 @@ namespace Ryujinx.Memory.Tracking
 
             for (int i = startHandle; i <= lastHandle; i++)
             {
-                _handles[i].SequenceNumber--;
+                if (_sequenceNumberBitmap.Clear(i))
+                {
+                    _uncheckedHandles++;
+                }
+
                 _handles[i].ForceDirty();
             }
         }
 
-        public IEnumerable<RegionHandle> GetHandles()
-        {
-            return _handles;
-        }
-
-        public void SignalWrite()
-        {
-            Dirty = true;
-        }
-
         public void QueryModified(Action<ulong, ulong> modifiedAction)
         {
             if (!Dirty)
@@ -123,33 +148,95 @@ namespace Ryujinx.Memory.Tracking
             QueryModified(Address, Size, modifiedAction);
         }
 
-        public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction)
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void ParseDirtyBits(long dirtyBits, ref int baseBit, ref int prevHandle, ref ulong rgStart, ref ulong rgSize, Action<ulong, ulong> modifiedAction)
         {
-            int startHandle = (int)((address - Address) / Granularity);
-            int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
-
-            ulong rgStart = _handles[startHandle].Address;
-            ulong rgSize = 0;
-
-            for (int i = startHandle; i <= lastHandle; i++)
+            while (dirtyBits != 0)
             {
-                RegionHandle handle = _handles[i];
+                int bit = BitOperations.TrailingZeroCount(dirtyBits);
+
+                dirtyBits &= ~(1L << bit);
+
+                int handleIndex = baseBit + bit;
+
+                RegionHandle handle = _handles[handleIndex];
+
+                if (handleIndex != prevHandle + 1)
+                {
+                    // Submit handles scanned until the gap as dirty
+                    if (rgSize != 0)
+                    {
+                        modifiedAction(rgStart, rgSize);
+                        rgSize = 0;
+                    }
+                    rgStart = handle.Address;
+                }
 
                 if (handle.Dirty)
                 {
                     rgSize += handle.Size;
                     handle.Reprotect();
                 }
-                else
+
+                prevHandle = handleIndex;
+            }
+
+            baseBit += ConcurrentBitmap.IntSize;
+        }
+
+        public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction)
+        {
+            int startHandle = (int)((address - Address) / Granularity);
+            int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
+
+            ulong rgStart = _handles[startHandle].Address;
+
+            if (startHandle == lastHandle)
+            {
+                RegionHandle handle = _handles[startHandle];
+
+                if (handle.Dirty)
                 {
-                    // Submit the region scanned so far as dirty
-                    if (rgSize != 0)
-                    {
-                        modifiedAction(rgStart, rgSize);
-                        rgSize = 0;
-                    }
-                    rgStart = handle.EndAddress;
+                    handle.Reprotect();
+                    modifiedAction(rgStart, handle.Size);
                 }
+
+                return;
+            }
+
+            ulong rgSize = 0;
+
+            long[] masks = _dirtyBitmap.Masks;
+
+            int startIndex = startHandle >> ConcurrentBitmap.IntShift;
+            int startBit = startHandle & ConcurrentBitmap.IntMask;
+            long startMask = -1L << startBit;
+
+            int endIndex = lastHandle >> ConcurrentBitmap.IntShift;
+            int endBit = lastHandle & ConcurrentBitmap.IntMask;
+            long endMask = (long)(ulong.MaxValue >> (ConcurrentBitmap.IntMask - endBit));
+
+            long startValue = Volatile.Read(ref masks[startIndex]);
+
+            int baseBit = startIndex << ConcurrentBitmap.IntShift;
+            int prevHandle = startHandle - 1;
+
+            if (startIndex == endIndex)
+            {
+                ParseDirtyBits(startValue & startMask & endMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
+            }
+            else
+            {
+                ParseDirtyBits(startValue & startMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
+
+                for (int i = startIndex + 1; i < endIndex; i++)
+                {
+                    ParseDirtyBits(Volatile.Read(ref masks[i]), ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
+                }
+
+                long endValue = Volatile.Read(ref masks[endIndex]);
+
+                ParseDirtyBits(endValue & endMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
             }
 
             if (rgSize != 0)
@@ -158,35 +245,120 @@ namespace Ryujinx.Memory.Tracking
             }
         }
 
-        public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction, int sequenceNumber)
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void ParseDirtyBits(long dirtyBits, long mask, int index, long[] seqMasks, ref int baseBit, ref int prevHandle, ref ulong rgStart, ref ulong rgSize, Action<ulong, ulong> modifiedAction)
         {
-            int startHandle = (int)((address - Address) / Granularity);
-            int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
+            long seqMask = mask & ~seqMasks[index];
+            dirtyBits &= seqMask;
 
-            ulong rgStart = _handles[startHandle].Address;
-            ulong rgSize = 0;
-
-            for (int i = startHandle; i <= lastHandle; i++)
+            while (dirtyBits != 0)
             {
-                RegionHandle handle = _handles[i];
+                int bit = BitOperations.TrailingZeroCount(dirtyBits);
 
-                if (sequenceNumber != handle.SequenceNumber && handle.DirtyOrVolatile())
+                dirtyBits &= ~(1L << bit);
+
+                int handleIndex = baseBit + bit;
+
+                RegionHandle handle = _handles[handleIndex];
+
+                if (handleIndex != prevHandle + 1)
                 {
-                    rgSize += handle.Size;
-                    handle.Reprotect();
-                }
-                else
-                {
-                    // Submit the region scanned so far as dirty
+                    // Submit handles scanned until the gap as dirty
                     if (rgSize != 0)
                     {
                         modifiedAction(rgStart, rgSize);
                         rgSize = 0;
                     }
-                    rgStart = handle.EndAddress;
+                    rgStart = handle.Address;
                 }
 
-                handle.SequenceNumber = sequenceNumber;
+                rgSize += handle.Size;
+                handle.Reprotect();
+
+                prevHandle = handleIndex;
+            }
+
+            seqMasks[index] |= mask;
+            _uncheckedHandles -= BitOperations.PopCount((ulong)seqMask);
+
+            baseBit += ConcurrentBitmap.IntSize;
+        }
+
+        public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction, int sequenceNumber)
+        {
+            int startHandle = (int)((address - Address) / Granularity);
+            int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
+
+            ulong rgStart = Address + (ulong)startHandle * Granularity;
+
+            if (sequenceNumber != _sequenceNumber)
+            {
+                if (_uncheckedHandles != _handles.Length)
+                {
+                    _sequenceNumberBitmap.Clear();
+                    _uncheckedHandles = _handles.Length;
+                }
+
+                _sequenceNumber = sequenceNumber;
+            }
+
+            if (startHandle == lastHandle)
+            {
+                var handle = _handles[startHandle];
+                if (_sequenceNumberBitmap.Set(startHandle))
+                {
+                    _uncheckedHandles--;
+
+                    if (handle.DirtyOrVolatile())
+                    {
+                        handle.Reprotect();
+
+                        modifiedAction(rgStart, handle.Size);
+                    }
+                }
+
+                return;
+            }
+
+            if (_uncheckedHandles == 0)
+            {
+                return;
+            }
+
+            ulong rgSize = 0;
+
+            long[] seqMasks = _sequenceNumberBitmap.Masks;
+            long[] masks = _dirtyBitmap.Masks;
+
+            int startIndex = startHandle >> ConcurrentBitmap.IntShift;
+            int startBit = startHandle & ConcurrentBitmap.IntMask;
+            long startMask = -1L << startBit;
+
+            int endIndex = lastHandle >> ConcurrentBitmap.IntShift;
+            int endBit = lastHandle & ConcurrentBitmap.IntMask;
+            long endMask = (long)(ulong.MaxValue >> (ConcurrentBitmap.IntMask - endBit));
+
+            long startValue = Volatile.Read(ref masks[startIndex]);
+
+            int baseBit = startIndex << ConcurrentBitmap.IntShift;
+            int prevHandle = startHandle - 1;
+
+            if (startIndex == endIndex)
+            {
+                ParseDirtyBits(startValue, startMask & endMask, startIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
+            }
+            else
+            {
+                ParseDirtyBits(startValue, startMask, startIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
+
+                for (int i = startIndex + 1; i < endIndex; i++)
+                {
+                    ParseDirtyBits(Volatile.Read(ref masks[i]), -1L, i, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
+                }
+
+                long endValue = Volatile.Read(ref masks[endIndex]);
+
+                ParseDirtyBits(endValue, endMask, endIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
             }
 
             if (rgSize != 0)

Ryujinx.Memory/Tracking/RegionHandle.cs

@@ -1,5 +1,4 @@
-using Ryujinx.Memory.Range;
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -10,7 +9,7 @@ namespace Ryujinx.Memory.Tracking
     /// A tracking handle for a given region of virtual memory. The Dirty flag is updated whenever any changes are made,
     /// and an action can be performed when the region is read to or written from.
     /// </summary>
-    public class RegionHandle : IRegionHandle, IRange
+    public class RegionHandle : IRegionHandle
     {
         /// <summary>
         /// If more than this number of checks have been performed on a dirty flag since its last reprotect,
@@ -23,7 +22,20 @@ namespace Ryujinx.Memory.Tracking
         /// </summary>
         private static int VolatileThreshold = 5;
 
-        public bool Dirty { get; private set; }
+        public bool Dirty
+        {
+            get
+            {
+                return Bitmap.IsSet(DirtyBit);
+            }
+            protected set
+            {
+                Bitmap.Set(DirtyBit, value);
+            }
+        }
+
+        internal int SequenceNumber { get; set; }
+
         public bool Unmapped { get; private set; }
 
         public ulong Address { get; }
@@ -31,7 +43,6 @@ namespace Ryujinx.Memory.Tracking
         public ulong EndAddress { get; }
 
         internal IMultiRegionHandle Parent { get; set; }
-        internal int SequenceNumber { get; set; }
 
         private event Action _onDirty;
 
@@ -68,17 +79,26 @@ namespace Ryujinx.Memory.Tracking
 
         internal RegionSignal PreAction => _preAction;
 
+        internal ConcurrentBitmap Bitmap;
+        internal int DirtyBit;
+
         /// <summary>
-        /// Create a new region handle. The handle is registered with the given tracking object,
+        /// Create a new bitmap backed region handle. The handle is registered with the given tracking object,
         /// and will be notified of any changes to the specified region.
         /// </summary>
         /// <param name="tracking">Tracking object for the target memory block</param>
         /// <param name="address">Virtual address of the region to track</param>
         /// <param name="size">Size of the region to track</param>
+        /// <param name="bitmap">The bitmap the dirty flag for this handle is stored in</param>
+        /// <param name="bit">The bit index representing the dirty flag for this handle</param>
         /// <param name="mapped">True if the region handle starts mapped</param>
-        internal RegionHandle(MemoryTracking tracking, ulong address, ulong size, bool mapped = true)
+        internal RegionHandle(MemoryTracking tracking, ulong address, ulong size, ConcurrentBitmap bitmap, int bit, bool mapped = true)
         {
+            Bitmap = bitmap;
+            DirtyBit = bit;
+
             Dirty = mapped;
+
             Unmapped = !mapped;
             Address = address;
             Size = size;
@@ -92,6 +112,54 @@ namespace Ryujinx.Memory.Tracking
             }
         }
 
+        /// <summary>
+        /// Create a new region handle. The handle is registered with the given tracking object,
+        /// and will be notified of any changes to the specified region.
+        /// </summary>
+        /// <param name="tracking">Tracking object for the target memory block</param>
+        /// <param name="address">Virtual address of the region to track</param>
+        /// <param name="size">Size of the region to track</param>
+        /// <param name="mapped">True if the region handle starts mapped</param>
+        internal RegionHandle(MemoryTracking tracking, ulong address, ulong size, bool mapped = true)
+        {
+            Bitmap = new ConcurrentBitmap(1, mapped);
+
+            Unmapped = !mapped;
+            Address = address;
+            Size = size;
+            EndAddress = address + size;
+
+            _tracking = tracking;
+            _regions = tracking.GetVirtualRegionsForHandle(address, size);
+            foreach (var region in _regions)
+            {
+                region.Handles.Add(this);
+            }
+        }
+
+        /// <summary>
+        /// Replace the bitmap and bit index used to track dirty state.
+        /// </summary>
+        /// <remarks>
+        /// The tracking lock should be held when this is called, to ensure neither bitmap is modified.
+        /// </remarks>
+        /// <param name="bitmap">The bitmap the dirty flag for this handle is stored in</param>
+        /// <param name="bit">The bit index representing the dirty flag for this handle</param>
+        internal void ReplaceBitmap(ConcurrentBitmap bitmap, int bit)
+        {
+            // Assumes the tracking lock is held, so nothing else can signal right now.
+
+            var oldBitmap = Bitmap;
+            var oldBit = DirtyBit;
+
+            bitmap.Set(bit, Dirty);
+
+            Bitmap = bitmap;
+            DirtyBit = bit;
+
+            Dirty |= oldBitmap.IsSet(oldBit);
+        }
+
         /// <summary>
         /// Clear the volatile state of this handle.
         /// </summary>
@@ -108,7 +176,7 @@ namespace Ryujinx.Memory.Tracking
         public bool DirtyOrVolatile()
         {
             _checkCount++;
-            return Dirty || _volatile;
+            return _volatile || Dirty;
         }
 
         /// <summary>
@@ -195,6 +263,8 @@ namespace Ryujinx.Memory.Tracking
         /// </summary>
         public void ForceDirty()
         {
+            _checkCount++;
+
             Dirty = true;
         }