Vulkan separate descriptor set fixes (#6895)

* Ensure descriptor sets are only re-used when all command buffers using it have completed

* Fix some SPIR-V capabilities

* Set update after bind flag if we exceed limits

* Simpler fix for Intel

* Format whitespace

* Make struct readonly

* Add barriers for extra set arrays too

src/Ryujinx.Graphics.GAL/IImageArray.cs

@@ -1,6 +1,8 @@
+using System;
+
 namespace Ryujinx.Graphics.GAL
 {
-    public interface IImageArray
+    public interface IImageArray : IDisposable
     {
         void SetFormats(int index, Format[] imageFormats);
         void SetImages(int index, ITexture[] images);

src/Ryujinx.Graphics.GAL/ITextureArray.cs

@@ -1,6 +1,8 @@
+using System;
+
 namespace Ryujinx.Graphics.GAL
 {
-    public interface ITextureArray
+    public interface ITextureArray : IDisposable
     {
         void SetSamplers(int index, ISampler[] samplers);
         void SetTextures(int index, ITexture[] textures);

src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs

@@ -66,6 +66,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
             Register<CounterEventDisposeCommand>(CommandType.CounterEventDispose);
             Register<CounterEventFlushCommand>(CommandType.CounterEventFlush);
 
+            Register<ImageArrayDisposeCommand>(CommandType.ImageArrayDispose);
             Register<ImageArraySetFormatsCommand>(CommandType.ImageArraySetFormats);
             Register<ImageArraySetImagesCommand>(CommandType.ImageArraySetImages);
 
@@ -88,6 +89,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
             Register<TextureSetDataSliceRegionCommand>(CommandType.TextureSetDataSliceRegion);
             Register<TextureSetStorageCommand>(CommandType.TextureSetStorage);
 
+            Register<TextureArrayDisposeCommand>(CommandType.TextureArrayDispose);
             Register<TextureArraySetSamplersCommand>(CommandType.TextureArraySetSamplers);
             Register<TextureArraySetTexturesCommand>(CommandType.TextureArraySetTextures);
 

src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs

@@ -26,6 +26,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
         CounterEventDispose,
         CounterEventFlush,
 
+        ImageArrayDispose,
         ImageArraySetFormats,
         ImageArraySetImages,
 
@@ -48,6 +49,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
         TextureSetDataSliceRegion,
         TextureSetStorage,
 
+        TextureArrayDispose,
         TextureArraySetSamplers,
         TextureArraySetTextures,
 

src/Ryujinx.Graphics.GAL/Multithreading/Commands/ImageArray/ImageArrayDisposeCommand.cs

@@ -0,0 +1,21 @@
+using Ryujinx.Graphics.GAL.Multithreading.Model;
+using Ryujinx.Graphics.GAL.Multithreading.Resources;
+
+namespace Ryujinx.Graphics.GAL.Multithreading.Commands.ImageArray
+{
+    struct ImageArrayDisposeCommand : IGALCommand, IGALCommand<ImageArrayDisposeCommand>
+    {
+        public readonly CommandType CommandType => CommandType.ImageArrayDispose;
+        private TableRef<ThreadedImageArray> _imageArray;
+
+        public void Set(TableRef<ThreadedImageArray> imageArray)
+        {
+            _imageArray = imageArray;
+        }
+
+        public static void Run(ref ImageArrayDisposeCommand command, ThreadedRenderer threaded, IRenderer renderer)
+        {
+            command._imageArray.Get(threaded).Base.Dispose();
+        }
+    }
+}

src/Ryujinx.Graphics.GAL/Multithreading/Commands/TextureArray/TextureArrayDisposeCommand.cs

@@ -0,0 +1,21 @@
+using Ryujinx.Graphics.GAL.Multithreading.Model;
+using Ryujinx.Graphics.GAL.Multithreading.Resources;
+
+namespace Ryujinx.Graphics.GAL.Multithreading.Commands.TextureArray
+{
+    struct TextureArrayDisposeCommand : IGALCommand, IGALCommand<TextureArrayDisposeCommand>
+    {
+        public readonly CommandType CommandType => CommandType.TextureArrayDispose;
+        private TableRef<ThreadedTextureArray> _textureArray;
+
+        public void Set(TableRef<ThreadedTextureArray> textureArray)
+        {
+            _textureArray = textureArray;
+        }
+
+        public static void Run(ref TextureArrayDisposeCommand command, ThreadedRenderer threaded, IRenderer renderer)
+        {
+            command._textureArray.Get(threaded).Base.Dispose();
+        }
+    }
+}

src/Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedImageArray.cs

@@ -21,6 +21,12 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
             return new TableRef<T>(_renderer, reference);
         }
 
+        public void Dispose()
+        {
+            _renderer.New<ImageArrayDisposeCommand>().Set(Ref(this));
+            _renderer.QueueCommand();
+        }
+
         public void SetFormats(int index, Format[] imageFormats)
         {
             _renderer.New<ImageArraySetFormatsCommand>().Set(Ref(this), index, Ref(imageFormats));

src/Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedTextureArray.cs

@@ -22,6 +22,12 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
             return new TableRef<T>(_renderer, reference);
         }
 
+        public void Dispose()
+        {
+            _renderer.New<TextureArrayDisposeCommand>().Set(Ref(this));
+            _renderer.QueueCommand();
+        }
+
         public void SetSamplers(int index, ISampler[] samplers)
         {
             _renderer.New<TextureArraySetSamplersCommand>().Set(Ref(this), index, Ref(samplers.ToArray()));

src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs

@@ -1113,6 +1113,15 @@ namespace Ryujinx.Graphics.Gpu.Image
                 nextNode = nextNode.Next;
                 _cacheFromBuffer.Remove(toRemove.Value.Key);
                 _lruCache.Remove(toRemove);
+
+                if (toRemove.Value.Key.IsImage)
+                {
+                    toRemove.Value.ImageArray.Dispose();
+                }
+                else
+                {
+                    toRemove.Value.TextureArray.Dispose();
+                }
             }
         }
 
@@ -1124,11 +1133,20 @@ namespace Ryujinx.Graphics.Gpu.Image
         {
             List<CacheEntryFromPoolKey> keysToRemove = null;
 
-            foreach (CacheEntryFromPoolKey key in _cacheFromPool.Keys)
+            foreach ((CacheEntryFromPoolKey key, CacheEntry entry) in _cacheFromPool)
             {
                 if (key.MatchesPool(pool))
                 {
                     (keysToRemove ??= new()).Add(key);
+
+                    if (key.IsImage)
+                    {
+                        entry.ImageArray.Dispose();
+                    }
+                    else
+                    {
+                        entry.TextureArray.Dispose();
+                    }
                 }
             }
 

src/Ryujinx.Graphics.OpenGL/Image/ImageArray.cs

@@ -63,5 +63,9 @@ namespace Ryujinx.Graphics.OpenGL.Image
                 }
             }
         }
+
+        public void Dispose()
+        {
+        }
     }
 }

src/Ryujinx.Graphics.OpenGL/Image/TextureArray.cs

@@ -48,5 +48,9 @@ namespace Ryujinx.Graphics.OpenGL.Image
                 }
             }
         }
+
+        public void Dispose()
+        {
+        }
     }
 }

src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs

@@ -98,11 +98,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
             Logger = parameters.Logger;
             TargetApi = parameters.TargetApi;
 
-            AddCapability(Capability.Shader);
-            AddCapability(Capability.Float64);
-
-            SetMemoryModel(AddressingModel.Logical, MemoryModel.GLSL450);
-
             Delegates = new SpirvDelegates(this);
         }
 

src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs

@@ -43,6 +43,10 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
 
             CodeGenContext context = new(info, parameters, instPool, integerPool);
 
+            context.AddCapability(Capability.Shader);
+
+            context.SetMemoryModel(AddressingModel.Logical, MemoryModel.GLSL450);
+
             context.AddCapability(Capability.GroupNonUniformBallot);
             context.AddCapability(Capability.GroupNonUniformShuffle);
             context.AddCapability(Capability.GroupNonUniformVote);
@@ -51,6 +55,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
             context.AddCapability(Capability.ImageQuery);
             context.AddCapability(Capability.SampledBuffer);
 
+            if (parameters.HostCapabilities.SupportsShaderFloat64)
+            {
+                context.AddCapability(Capability.Float64);
+            }
+
             if (parameters.Definitions.TransformFeedbackEnabled && parameters.Definitions.LastInVertexPipeline)
             {
                 context.AddCapability(Capability.TransformFeedback);
@@ -58,7 +67,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
 
             if (parameters.Definitions.Stage == ShaderStage.Fragment)
             {
-                if (context.Info.IoDefinitions.Contains(new IoDefinition(StorageKind.Input, IoVariable.Layer)))
+                if (context.Info.IoDefinitions.Contains(new IoDefinition(StorageKind.Input, IoVariable.Layer)) ||
+                    context.Info.IoDefinitions.Contains(new IoDefinition(StorageKind.Input, IoVariable.PrimitiveId)))
                 {
                     context.AddCapability(Capability.Geometry);
                 }

src/Ryujinx.Graphics.Shader/Translation/HostCapabilities.cs

@@ -8,6 +8,7 @@ namespace Ryujinx.Graphics.Shader.Translation
         public readonly bool SupportsGeometryShaderPassthrough;
         public readonly bool SupportsShaderBallot;
         public readonly bool SupportsShaderBarrierDivergence;
+        public readonly bool SupportsShaderFloat64;
         public readonly bool SupportsTextureShadowLod;
         public readonly bool SupportsViewportMask;
 
@@ -18,6 +19,7 @@ namespace Ryujinx.Graphics.Shader.Translation
             bool supportsGeometryShaderPassthrough,
             bool supportsShaderBallot,
             bool supportsShaderBarrierDivergence,
+            bool supportsShaderFloat64,
             bool supportsTextureShadowLod,
             bool supportsViewportMask)
         {
@@ -27,6 +29,7 @@ namespace Ryujinx.Graphics.Shader.Translation
             SupportsGeometryShaderPassthrough = supportsGeometryShaderPassthrough;
             SupportsShaderBallot = supportsShaderBallot;
             SupportsShaderBarrierDivergence = supportsShaderBarrierDivergence;
+            SupportsShaderFloat64 = supportsShaderFloat64;
             SupportsTextureShadowLod = supportsTextureShadowLod;
             SupportsViewportMask = supportsViewportMask;
         }

src/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs

@@ -363,6 +363,7 @@ namespace Ryujinx.Graphics.Shader.Translation
                 GpuAccessor.QueryHostSupportsGeometryShaderPassthrough(),
                 GpuAccessor.QueryHostSupportsShaderBallot(),
                 GpuAccessor.QueryHostSupportsShaderBarrierDivergence(),
+                GpuAccessor.QueryHostSupportsShaderFloat64(),
                 GpuAccessor.QueryHostSupportsTextureShadowLod(),
                 GpuAccessor.QueryHostSupportsViewportMask());
 

src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs

@@ -291,8 +291,9 @@ namespace Ryujinx.Graphics.Vulkan
                     }
                     else
                     {
-                        PipelineStageFlags stageFlags = _textureArrayRefs[segment.Binding].Stage.ConvertToPipelineStageFlags();
+                        ref var arrayRef = ref _textureArrayRefs[segment.Binding];
-                        _textureArrayRefs[segment.Binding].Array?.QueueWriteToReadBarriers(cbs, stageFlags);
+                        PipelineStageFlags stageFlags = arrayRef.Stage.ConvertToPipelineStageFlags();
+                        arrayRef.Array?.QueueWriteToReadBarriers(cbs, stageFlags);
                     }
                 }
             }
@@ -311,8 +312,40 @@ namespace Ryujinx.Graphics.Vulkan
                     }
                     else
                     {
-                        PipelineStageFlags stageFlags = _imageArrayRefs[segment.Binding].Stage.ConvertToPipelineStageFlags();
+                        ref var arrayRef = ref _imageArrayRefs[segment.Binding];
-                        _imageArrayRefs[segment.Binding].Array?.QueueWriteToReadBarriers(cbs, stageFlags);
+                        PipelineStageFlags stageFlags = arrayRef.Stage.ConvertToPipelineStageFlags();
+                        arrayRef.Array?.QueueWriteToReadBarriers(cbs, stageFlags);
+                    }
+                }
+            }
+
+            for (int setIndex = PipelineBase.DescriptorSetLayouts; setIndex < _program.BindingSegments.Length; setIndex++)
+            {
+                var bindingSegments = _program.BindingSegments[setIndex];
+
+                if (bindingSegments.Length == 0)
+                {
+                    continue;
+                }
+
+                ResourceBindingSegment segment = bindingSegments[0];
+
+                if (segment.IsArray)
+                {
+                    if (segment.Type == ResourceType.Texture ||
+                        segment.Type == ResourceType.Sampler ||
+                        segment.Type == ResourceType.TextureAndSampler ||
+                        segment.Type == ResourceType.BufferTexture)
+                    {
+                        ref var arrayRef = ref _textureArrayExtraRefs[setIndex - PipelineBase.DescriptorSetLayouts];
+                        PipelineStageFlags stageFlags = arrayRef.Stage.ConvertToPipelineStageFlags();
+                        arrayRef.Array?.QueueWriteToReadBarriers(cbs, stageFlags);
+                    }
+                    else if (segment.Type == ResourceType.Image || segment.Type == ResourceType.BufferImage)
+                    {
+                        ref var arrayRef = ref _imageArrayExtraRefs[setIndex - PipelineBase.DescriptorSetLayouts];
+                        PipelineStageFlags stageFlags = arrayRef.Stage.ConvertToPipelineStageFlags();
+                        arrayRef.Array?.QueueWriteToReadBarriers(cbs, stageFlags);
                     }
                 }
             }

src/Ryujinx.Graphics.Vulkan/ImageArray.cs

@@ -2,11 +2,10 @@ using Ryujinx.Graphics.GAL;
 using Silk.NET.Vulkan;
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 
 namespace Ryujinx.Graphics.Vulkan
 {
-    class ImageArray : IImageArray
+    class ImageArray : ResourceArray, IImageArray
     {
         private readonly VulkanRenderer _gd;
 
@@ -25,19 +24,11 @@ namespace Ryujinx.Graphics.Vulkan
 
         private HashSet<TextureStorage> _storages;
 
-        private DescriptorSet[] _cachedDescriptorSets;
-
         private int _cachedCommandBufferIndex;
         private int _cachedSubmissionCount;
 
-        private ShaderCollection _cachedDscProgram;
-        private int _cachedDscSetIndex;
-        private int _cachedDscIndex;
-
         private readonly bool _isBuffer;
 
-        private int _bindCount;
-
         public ImageArray(VulkanRenderer gd, int size, bool isBuffer)
         {
             _gd = gd;
@@ -104,12 +95,7 @@ namespace Ryujinx.Graphics.Vulkan
         {
             _cachedCommandBufferIndex = -1;
             _storages = null;
-            _cachedDescriptorSets = null;
+            SetDirty(_gd);
-
-            if (_bindCount != 0)
-            {
-                _gd.PipelineInternal.ForceImageDirty();
-            }
         }
 
         public void QueueWriteToReadBarriers(CommandBufferScoped cbs, PipelineStageFlags stageFlags)
@@ -195,7 +181,7 @@ namespace Ryujinx.Graphics.Vulkan
             int setIndex,
             TextureView dummyTexture)
         {
-            if (_cachedDescriptorSets != null)
+            if (TryGetCachedDescriptorSets(cbs, program, setIndex, out DescriptorSet[] sets))
             {
                 // We still need to ensure the current command buffer holds a reference to all used textures.
 
@@ -208,12 +194,9 @@ namespace Ryujinx.Graphics.Vulkan
                     GetBufferViews(cbs);
                 }
 
-                return _cachedDescriptorSets;
+                return sets;
             }
 
-            _cachedDscProgram?.ReleaseManualDescriptorSetCollection(_cachedDscSetIndex, _cachedDscIndex);
-            var dsc = program.GetNewManualDescriptorSetCollection(cbs.CommandBufferIndex, setIndex, out _cachedDscIndex).Get(cbs);
-
             DescriptorSetTemplate template = program.Templates[setIndex];
 
             DescriptorSetTemplateWriter tu = templateUpdater.Begin(template);
@@ -227,24 +210,9 @@ namespace Ryujinx.Graphics.Vulkan
                 tu.Push(GetBufferViews(cbs));
             }
 
-            var sets = dsc.GetSets();
             templateUpdater.Commit(_gd, device, sets[0]);
-            _cachedDescriptorSets = sets;
-            _cachedDscProgram = program;
-            _cachedDscSetIndex = setIndex;
 
             return sets;
         }
-
-        public void IncrementBindCount()
-        {
-            _bindCount++;
-        }
-
-        public void DecrementBindCount()
-        {
-            int newBindCount = --_bindCount;
-            Debug.Assert(newBindCount >= 0);
-        }
     }
 }

src/Ryujinx.Graphics.Vulkan/PipelineLayoutCacheEntry.cs

@@ -3,6 +3,7 @@ using Silk.NET.Vulkan;
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Diagnostics;
 using System.Runtime.InteropServices;
 
 namespace Ryujinx.Graphics.Vulkan
@@ -15,6 +16,7 @@ namespace Ryujinx.Graphics.Vulkan
         private readonly Device _device;
 
         public DescriptorSetLayout[] DescriptorSetLayouts { get; }
+        public bool[] DescriptorSetLayoutsUpdateAfterBind { get; }
         public PipelineLayout PipelineLayout { get; }
 
         private readonly int[] _consumedDescriptorsPerSet;
@@ -31,20 +33,37 @@ namespace Ryujinx.Graphics.Vulkan
         private struct ManualDescriptorSetEntry
         {
             public Auto<DescriptorSetCollection> DescriptorSet;
-            public int CbIndex;
+            public uint CbRefMask;
-            public int CbSubmissionCount;
             public bool InUse;
 
-            public ManualDescriptorSetEntry(Auto<DescriptorSetCollection> descriptorSet, int cbIndex, int cbSubmissionCount, bool inUse)
+            public ManualDescriptorSetEntry(Auto<DescriptorSetCollection> descriptorSet, int cbIndex)
             {
                 DescriptorSet = descriptorSet;
-                CbIndex = cbIndex;
+                CbRefMask = 1u << cbIndex;
-                CbSubmissionCount = cbSubmissionCount;
+                InUse = true;
-                InUse = inUse;
+            }
+        }
+
+        private readonly struct PendingManualDsConsumption
+        {
+            public FenceHolder Fence { get; }
+            public int CommandBufferIndex { get; }
+            public int SetIndex { get; }
+            public int CacheIndex { get; }
+
+            public PendingManualDsConsumption(FenceHolder fence, int commandBufferIndex, int setIndex, int cacheIndex)
+            {
+                Fence = fence;
+                CommandBufferIndex = commandBufferIndex;
+                SetIndex = setIndex;
+                CacheIndex = cacheIndex;
+                fence.Get();
             }
         }
 
         private readonly List<ManualDescriptorSetEntry>[] _manualDsCache;
+        private readonly Queue<PendingManualDsConsumption> _pendingManualDsConsumptions;
+        private readonly Queue<int>[] _freeManualDsCacheEntries;
 
         private readonly Dictionary<long, DescriptorSetTemplate> _pdTemplates;
         private readonly ResourceDescriptorCollection _pdDescriptors;
@@ -70,6 +89,8 @@ namespace Ryujinx.Graphics.Vulkan
 
             _dsCacheCursor = new int[setsCount];
             _manualDsCache = new List<ManualDescriptorSetEntry>[setsCount];
+            _pendingManualDsConsumptions = new Queue<PendingManualDsConsumption>();
+            _freeManualDsCacheEntries = new Queue<int>[setsCount];
         }
 
         public PipelineLayoutCacheEntry(
@@ -78,7 +99,11 @@ namespace Ryujinx.Graphics.Vulkan
             ReadOnlyCollection<ResourceDescriptorCollection> setDescriptors,
             bool usePushDescriptors) : this(gd, device, setDescriptors.Count)
         {
-            (DescriptorSetLayouts, PipelineLayout) = PipelineLayoutFactory.Create(gd, device, setDescriptors, usePushDescriptors);
+            ResourceLayouts layouts = PipelineLayoutFactory.Create(gd, device, setDescriptors, usePushDescriptors);
+
+            DescriptorSetLayouts = layouts.DescriptorSetLayouts;
+            DescriptorSetLayoutsUpdateAfterBind = layouts.DescriptorSetLayoutsUpdateAfterBind;
+            PipelineLayout = layouts.PipelineLayout;
 
             _consumedDescriptorsPerSet = new int[setDescriptors.Count];
             _poolSizes = new DescriptorPoolSize[setDescriptors.Count][];
@@ -133,7 +158,7 @@ namespace Ryujinx.Graphics.Vulkan
                     _poolSizes[setIndex],
                     setIndex,
                     _consumedDescriptorsPerSet[setIndex],
-                    false);
+                    DescriptorSetLayoutsUpdateAfterBind[setIndex]);
 
                 list.Add(dsc);
                 isNew = true;
@@ -144,49 +169,99 @@ namespace Ryujinx.Graphics.Vulkan
             return list[index];
         }
 
-        public Auto<DescriptorSetCollection> GetNewManualDescriptorSetCollection(int commandBufferIndex, int setIndex, out int cacheIndex)
+        public Auto<DescriptorSetCollection> GetNewManualDescriptorSetCollection(CommandBufferScoped cbs, int setIndex, out int cacheIndex)
         {
-            int submissionCount = _gd.CommandBufferPool.GetSubmissionCount(commandBufferIndex);
+            FreeCompletedManualDescriptorSets();
 
             var list = _manualDsCache[setIndex] ??= new();
             var span = CollectionsMarshal.AsSpan(list);
 
-            for (int index = 0; index < span.Length; index++)
+            Queue<int> freeQueue = _freeManualDsCacheEntries[setIndex];
-            {
-                ref ManualDescriptorSetEntry entry = ref span[index];
 
-                if (!entry.InUse && (entry.CbIndex != commandBufferIndex || entry.CbSubmissionCount != submissionCount))
+            // Do we have at least one freed descriptor set? If so, just use that.
+            if (freeQueue != null && freeQueue.TryDequeue(out int freeIndex))
             {
+                ref ManualDescriptorSetEntry entry = ref span[freeIndex];
+
+                Debug.Assert(!entry.InUse && entry.CbRefMask == 0);
+
                 entry.InUse = true;
-                    entry.CbIndex = commandBufferIndex;
+                entry.CbRefMask = 1u << cbs.CommandBufferIndex;
-                    entry.CbSubmissionCount = submissionCount;
+                cacheIndex = freeIndex;
 
-                    cacheIndex = index;
+                _pendingManualDsConsumptions.Enqueue(new PendingManualDsConsumption(cbs.GetFence(), cbs.CommandBufferIndex, setIndex, freeIndex));
 
                 return entry.DescriptorSet;
             }
-            }
 
+            // Otherwise create a new descriptor set, and add to our pending queue for command buffer consumption tracking.
             var dsc = _descriptorSetManager.AllocateDescriptorSet(
                 _gd.Api,
                 DescriptorSetLayouts[setIndex],
                 _poolSizes[setIndex],
                 setIndex,
                 _consumedDescriptorsPerSet[setIndex],
-                false);
+                DescriptorSetLayoutsUpdateAfterBind[setIndex]);
 
             cacheIndex = list.Count;
-            list.Add(new ManualDescriptorSetEntry(dsc, commandBufferIndex, submissionCount, inUse: true));
+            list.Add(new ManualDescriptorSetEntry(dsc, cbs.CommandBufferIndex));
+            _pendingManualDsConsumptions.Enqueue(new PendingManualDsConsumption(cbs.GetFence(), cbs.CommandBufferIndex, setIndex, cacheIndex));
 
             return dsc;
         }
 
+        public void UpdateManualDescriptorSetCollectionOwnership(CommandBufferScoped cbs, int setIndex, int cacheIndex)
+        {
+            FreeCompletedManualDescriptorSets();
+
+            var list = _manualDsCache[setIndex];
+            var span = CollectionsMarshal.AsSpan(list);
+            ref var entry = ref span[cacheIndex];
+
+            uint cbMask = 1u << cbs.CommandBufferIndex;
+
+            if ((entry.CbRefMask & cbMask) == 0)
+            {
+                entry.CbRefMask |= cbMask;
+
+                _pendingManualDsConsumptions.Enqueue(new PendingManualDsConsumption(cbs.GetFence(), cbs.CommandBufferIndex, setIndex, cacheIndex));
+            }
+        }
+
+        private void FreeCompletedManualDescriptorSets()
+        {
+            FenceHolder signalledFence = null;
+            while (_pendingManualDsConsumptions.TryPeek(out var pds) && (pds.Fence == signalledFence || pds.Fence.IsSignaled()))
+            {
+                signalledFence = pds.Fence; // Already checked - don't need to do it again.
+                var dequeued = _pendingManualDsConsumptions.Dequeue();
+                Debug.Assert(dequeued.Fence == pds.Fence);
+                pds.Fence.Put();
+
+                var span = CollectionsMarshal.AsSpan(_manualDsCache[dequeued.SetIndex]);
+                ref var entry = ref span[dequeued.CacheIndex];
+                entry.CbRefMask &= ~(1u << dequeued.CommandBufferIndex);
+
+                if (!entry.InUse && entry.CbRefMask == 0)
+                {
+                    // If not in use by any array, and not bound to any command buffer, the descriptor set can be re-used immediately.
+                    (_freeManualDsCacheEntries[dequeued.SetIndex] ??= new()).Enqueue(dequeued.CacheIndex);
+                }
+            }
+        }
+
         public void ReleaseManualDescriptorSetCollection(int setIndex, int cacheIndex)
         {
             var list = _manualDsCache[setIndex];
             var span = CollectionsMarshal.AsSpan(list);
 
             span[cacheIndex].InUse = false;
+
+            if (span[cacheIndex].CbRefMask == 0)
+            {
+                // This is no longer in use by any array, so if not bound to any command buffer, the descriptor set can be re-used immediately.
+                (_freeManualDsCacheEntries[setIndex] ??= new()).Enqueue(cacheIndex);
+            }
         }
 
         private static Span<DescriptorPoolSize> GetDescriptorPoolSizes(Span<DescriptorPoolSize> output, ResourceDescriptorCollection setDescriptor, uint multiplier)
@@ -291,6 +366,11 @@ namespace Ryujinx.Graphics.Vulkan
                     _gd.Api.DestroyDescriptorSetLayout(_device, DescriptorSetLayouts[i], null);
                 }
 
+                while (_pendingManualDsConsumptions.TryDequeue(out var pds))
+                {
+                    pds.Fence.Put();
+                }
+
                 _descriptorSetManager.Dispose();
             }
         }

src/Ryujinx.Graphics.Vulkan/PipelineLayoutFactory.cs

@@ -1,18 +1,23 @@
+using Ryujinx.Common.Memory;
 using Ryujinx.Graphics.GAL;
 using Silk.NET.Vulkan;
+using System;
 using System.Collections.ObjectModel;
 
 namespace Ryujinx.Graphics.Vulkan
 {
+    record struct ResourceLayouts(DescriptorSetLayout[] DescriptorSetLayouts, bool[] DescriptorSetLayoutsUpdateAfterBind, PipelineLayout PipelineLayout);
+
     static class PipelineLayoutFactory
     {
-        public static unsafe (DescriptorSetLayout[], PipelineLayout) Create(
+        public static unsafe ResourceLayouts Create(
             VulkanRenderer gd,
             Device device,
             ReadOnlyCollection<ResourceDescriptorCollection> setDescriptors,
             bool usePushDescriptors)
         {
             DescriptorSetLayout[] layouts = new DescriptorSetLayout[setDescriptors.Count];
+            bool[] updateAfterBindFlags = new bool[setDescriptors.Count];
 
             bool isMoltenVk = gd.IsMoltenVk;
 
@@ -32,10 +37,11 @@ namespace Ryujinx.Graphics.Vulkan
 
                 DescriptorSetLayoutBinding[] layoutBindings = new DescriptorSetLayoutBinding[rdc.Descriptors.Count];
 
+                bool hasArray = false;
+
                 for (int descIndex = 0; descIndex < rdc.Descriptors.Count; descIndex++)
                 {
                     ResourceDescriptor descriptor = rdc.Descriptors[descIndex];
-
                     ResourceStages stages = descriptor.Stages;
 
                     if (descriptor.Type == ResourceType.StorageBuffer && isMoltenVk)
@@ -52,16 +58,37 @@ namespace Ryujinx.Graphics.Vulkan
                         DescriptorCount = (uint)descriptor.Count,
                         StageFlags = stages.Convert(),
                     };
+
+                    if (descriptor.Count > 1)
+                    {
+                        hasArray = true;
+                    }
                 }
 
                 fixed (DescriptorSetLayoutBinding* pLayoutBindings = layoutBindings)
                 {
+                    DescriptorSetLayoutCreateFlags flags = DescriptorSetLayoutCreateFlags.None;
+
+                    if (usePushDescriptors && setIndex == 0)
+                    {
+                        flags = DescriptorSetLayoutCreateFlags.PushDescriptorBitKhr;
+                    }
+
+                    if (gd.Vendor == Vendor.Intel && hasArray)
+                    {
+                        // Some vendors (like Intel) have low per-stage limits.
+                        // We must set the flag if we exceed those limits.
+                        flags |= DescriptorSetLayoutCreateFlags.UpdateAfterBindPoolBit;
+
+                        updateAfterBindFlags[setIndex] = true;
+                    }
+
                     var descriptorSetLayoutCreateInfo = new DescriptorSetLayoutCreateInfo
                     {
                         SType = StructureType.DescriptorSetLayoutCreateInfo,
                         PBindings = pLayoutBindings,
                         BindingCount = (uint)layoutBindings.Length,
-                        Flags = usePushDescriptors && setIndex == 0 ? DescriptorSetLayoutCreateFlags.PushDescriptorBitKhr : DescriptorSetLayoutCreateFlags.None,
+                        Flags = flags,
                     };
 
                     gd.Api.CreateDescriptorSetLayout(device, descriptorSetLayoutCreateInfo, null, out layouts[setIndex]).ThrowOnError();
@@ -82,7 +109,7 @@ namespace Ryujinx.Graphics.Vulkan
                 gd.Api.CreatePipelineLayout(device, &pipelineLayoutCreateInfo, null, out layout).ThrowOnError();
             }
 
-            return (layouts, layout);
+            return new ResourceLayouts(layouts, updateAfterBindFlags, layout);
         }
     }
 }

src/Ryujinx.Graphics.Vulkan/ResourceArray.cs

@@ -0,0 +1,74 @@
+using Silk.NET.Vulkan;
+using System;
+using System.Diagnostics;
+
+namespace Ryujinx.Graphics.Vulkan
+{
+    class ResourceArray : IDisposable
+    {
+        private DescriptorSet[] _cachedDescriptorSets;
+
+        private ShaderCollection _cachedDscProgram;
+        private int _cachedDscSetIndex;
+        private int _cachedDscIndex;
+
+        private int _bindCount;
+
+        protected void SetDirty(VulkanRenderer gd)
+        {
+            ReleaseDescriptorSet();
+
+            if (_bindCount != 0)
+            {
+                gd.PipelineInternal.ForceTextureDirty();
+            }
+        }
+
+        public bool TryGetCachedDescriptorSets(CommandBufferScoped cbs, ShaderCollection program, int setIndex, out DescriptorSet[] sets)
+        {
+            if (_cachedDescriptorSets != null)
+            {
+                _cachedDscProgram.UpdateManualDescriptorSetCollectionOwnership(cbs, _cachedDscSetIndex, _cachedDscIndex);
+
+                sets = _cachedDescriptorSets;
+
+                return true;
+            }
+
+            var dsc = program.GetNewManualDescriptorSetCollection(cbs, setIndex, out _cachedDscIndex).Get(cbs);
+
+            sets = dsc.GetSets();
+
+            _cachedDescriptorSets = sets;
+            _cachedDscProgram = program;
+            _cachedDscSetIndex = setIndex;
+
+            return false;
+        }
+
+        public void IncrementBindCount()
+        {
+            _bindCount++;
+        }
+
+        public void DecrementBindCount()
+        {
+            int newBindCount = --_bindCount;
+            Debug.Assert(newBindCount >= 0);
+        }
+
+        private void ReleaseDescriptorSet()
+        {
+            if (_cachedDescriptorSets != null)
+            {
+                _cachedDscProgram.ReleaseManualDescriptorSetCollection(_cachedDscSetIndex, _cachedDscIndex);
+                _cachedDescriptorSets = null;
+            }
+        }
+
+        public void Dispose()
+        {
+            ReleaseDescriptorSet();
+        }
+    }
+}

src/Ryujinx.Graphics.Vulkan/ShaderCollection.cs

@@ -604,9 +604,14 @@ namespace Ryujinx.Graphics.Vulkan
             return _plce.GetNewDescriptorSetCollection(setIndex, out isNew);
         }
 
-        public Auto<DescriptorSetCollection> GetNewManualDescriptorSetCollection(int commandBufferIndex, int setIndex, out int cacheIndex)
+        public Auto<DescriptorSetCollection> GetNewManualDescriptorSetCollection(CommandBufferScoped cbs, int setIndex, out int cacheIndex)
         {
-            return _plce.GetNewManualDescriptorSetCollection(commandBufferIndex, setIndex, out cacheIndex);
+            return _plce.GetNewManualDescriptorSetCollection(cbs, setIndex, out cacheIndex);
+        }
+
+        public void UpdateManualDescriptorSetCollectionOwnership(CommandBufferScoped cbs, int setIndex, int cacheIndex)
+        {
+            _plce.UpdateManualDescriptorSetCollectionOwnership(cbs, setIndex, cacheIndex);
         }
 
         public void ReleaseManualDescriptorSetCollection(int setIndex, int cacheIndex)

src/Ryujinx.Graphics.Vulkan/TextureArray.cs

@@ -2,11 +2,10 @@ using Ryujinx.Graphics.GAL;
 using Silk.NET.Vulkan;
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 
 namespace Ryujinx.Graphics.Vulkan
 {
-    class TextureArray : ITextureArray
+    class TextureArray : ResourceArray, ITextureArray
     {
         private readonly VulkanRenderer _gd;
 
@@ -25,19 +24,11 @@ namespace Ryujinx.Graphics.Vulkan
 
         private HashSet<TextureStorage> _storages;
 
-        private DescriptorSet[] _cachedDescriptorSets;
-
         private int _cachedCommandBufferIndex;
         private int _cachedSubmissionCount;
 
-        private ShaderCollection _cachedDscProgram;
-        private int _cachedDscSetIndex;
-        private int _cachedDscIndex;
-
         private readonly bool _isBuffer;
 
-        private int _bindCount;
-
         public TextureArray(VulkanRenderer gd, int size, bool isBuffer)
         {
             _gd = gd;
@@ -113,12 +104,7 @@ namespace Ryujinx.Graphics.Vulkan
         {
             _cachedCommandBufferIndex = -1;
             _storages = null;
-            _cachedDescriptorSets = null;
+            SetDirty(_gd);
-
-            if (_bindCount != 0)
-            {
-                _gd.PipelineInternal.ForceTextureDirty();
-            }
         }
 
         public void QueueWriteToReadBarriers(CommandBufferScoped cbs, PipelineStageFlags stageFlags)
@@ -211,7 +197,7 @@ namespace Ryujinx.Graphics.Vulkan
             TextureView dummyTexture,
             SamplerHolder dummySampler)
         {
-            if (_cachedDescriptorSets != null)
+            if (TryGetCachedDescriptorSets(cbs, program, setIndex, out DescriptorSet[] sets))
             {
                 // We still need to ensure the current command buffer holds a reference to all used textures.
 
@@ -224,12 +210,9 @@ namespace Ryujinx.Graphics.Vulkan
                     GetBufferViews(cbs);
                 }
 
-                return _cachedDescriptorSets;
+                return sets;
             }
 
-            _cachedDscProgram?.ReleaseManualDescriptorSetCollection(_cachedDscSetIndex, _cachedDscIndex);
-            var dsc = program.GetNewManualDescriptorSetCollection(cbs.CommandBufferIndex, setIndex, out _cachedDscIndex).Get(cbs);
-
             DescriptorSetTemplate template = program.Templates[setIndex];
 
             DescriptorSetTemplateWriter tu = templateUpdater.Begin(template);
@@ -243,24 +226,9 @@ namespace Ryujinx.Graphics.Vulkan
                 tu.Push(GetBufferViews(cbs));
             }
 
-            var sets = dsc.GetSets();
             templateUpdater.Commit(_gd, device, sets[0]);
-            _cachedDescriptorSets = sets;
-            _cachedDscProgram = program;
-            _cachedDscSetIndex = setIndex;
 
             return sets;
         }
-
-        public void IncrementBindCount()
-        {
-            _bindCount++;
-        }
-
-        public void DecrementBindCount()
-        {
-            int newBindCount = --_bindCount;
-            Debug.Assert(newBindCount >= 0);
-        }
     }
 }