Support multiple destination operands on shader IR and shuffle predicates (#1964)

* Support multiple destination operands on shader IR and shuffle predicates

* Cache version change

Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs

@@ -34,7 +34,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
         /// <summary>
         /// Version of the codegen (to be changed when codegen or guest format change).
         /// </summary>
-        private const ulong ShaderCodeGenVersion = 1790;
+        private const ulong ShaderCodeGenVersion = 1964;
 
         /// <summary>
         /// Creates a new instance of the shader cache.

Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/Shuffle.glsl

@@ -1,9 +1,10 @@
-float Helper_Shuffle(float x, uint index, uint mask)
+float Helper_Shuffle(float x, uint index, uint mask, out bool valid)
 {
     uint clamp = mask & 0x1fu;
     uint segMask = (mask >> 8) & 0x1fu;
     uint minThreadId = gl_SubGroupInvocationARB & segMask;
     uint maxThreadId = minThreadId | (clamp & ~segMask);
     uint srcThreadId = (index & ~segMask) | minThreadId;
-    return (srcThreadId <= maxThreadId) ? readInvocationARB(x, srcThreadId) : x;
+    valid = srcThreadId <= maxThreadId;
+    return valid ? readInvocationARB(x, srcThreadId) : x;
 }

Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/ShuffleDown.glsl

@@ -1,9 +1,10 @@
-float Helper_ShuffleDown(float x, uint index, uint mask)
+float Helper_ShuffleDown(float x, uint index, uint mask, out bool valid)
 {
     uint clamp = mask & 0x1fu;
     uint segMask = (mask >> 8) & 0x1fu;
     uint minThreadId = gl_SubGroupInvocationARB & segMask;
     uint maxThreadId = minThreadId | (clamp & ~segMask);
     uint srcThreadId = gl_SubGroupInvocationARB + index;
-    return (srcThreadId <= maxThreadId) ? readInvocationARB(x, srcThreadId) : x;
+    valid = srcThreadId <= maxThreadId;
+    return valid ? readInvocationARB(x, srcThreadId) : x;
 }

Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/ShuffleUp.glsl

@@ -1,8 +1,9 @@
-float Helper_ShuffleUp(float x, uint index, uint mask)
+float Helper_ShuffleUp(float x, uint index, uint mask, out bool valid)
 {
     uint clamp = mask & 0x1fu;
     uint segMask = (mask >> 8) & 0x1fu;
     uint minThreadId = gl_SubGroupInvocationARB & segMask;
     uint srcThreadId = gl_SubGroupInvocationARB - index;
-    return (srcThreadId >= minThreadId) ? readInvocationARB(x, srcThreadId) : x;
+    valid = srcThreadId >= minThreadId;
+    return valid ? readInvocationARB(x, srcThreadId) : x;
 }

Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/ShuffleXor.glsl

@@ -1,9 +1,10 @@
-float Helper_ShuffleXor(float x, uint index, uint mask)
+float Helper_ShuffleXor(float x, uint index, uint mask, out bool valid)
 {
     uint clamp = mask & 0x1fu;
     uint segMask = (mask >> 8) & 0x1fu;
     uint minThreadId = gl_SubGroupInvocationARB & segMask;
     uint maxThreadId = minThreadId | (clamp & ~segMask);
     uint srcThreadId = gl_SubGroupInvocationARB ^ index;
-    return (srcThreadId <= maxThreadId) ? readInvocationARB(x, srcThreadId) : x;
+    valid = srcThreadId <= maxThreadId;
+    return valid ? readInvocationARB(x, srcThreadId) : x;
 }

Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs

@@ -88,13 +88,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.LoopContinue,             InstType.OpNullary,      "continue");
             Add(Instruction.PackDouble2x32,           InstType.Special);
             Add(Instruction.PackHalf2x16,             InstType.Special);
-            Add(Instruction.ShiftLeft,                InstType.OpBinary,       "<<",              3);
-            Add(Instruction.ShiftRightS32,            InstType.OpBinary,       ">>",              3);
-            Add(Instruction.ShiftRightU32,            InstType.OpBinary,       ">>",              3);
-            Add(Instruction.Shuffle,                  InstType.CallTernary,    HelperFunctionNames.Shuffle);
-            Add(Instruction.ShuffleDown,              InstType.CallTernary,    HelperFunctionNames.ShuffleDown);
-            Add(Instruction.ShuffleUp,                InstType.CallTernary,    HelperFunctionNames.ShuffleUp);
-            Add(Instruction.ShuffleXor,               InstType.CallTernary,    HelperFunctionNames.ShuffleXor);
             Add(Instruction.Maximum,                  InstType.CallBinary,     "max");
             Add(Instruction.MaximumU32,               InstType.CallBinary,     "max");
             Add(Instruction.MemoryBarrier,            InstType.CallNullary,    "memoryBarrier");
@@ -107,6 +100,13 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.ReciprocalSquareRoot,     InstType.CallUnary,      "inversesqrt");
             Add(Instruction.Return,                   InstType.OpNullary,      "return");
             Add(Instruction.Round,                    InstType.CallUnary,      "roundEven");
+            Add(Instruction.ShiftLeft,                InstType.OpBinary,       "<<",              3);
+            Add(Instruction.ShiftRightS32,            InstType.OpBinary,       ">>",              3);
+            Add(Instruction.ShiftRightU32,            InstType.OpBinary,       ">>",              3);
+            Add(Instruction.Shuffle,                  InstType.CallQuaternary, HelperFunctionNames.Shuffle);
+            Add(Instruction.ShuffleDown,              InstType.CallQuaternary, HelperFunctionNames.ShuffleDown);
+            Add(Instruction.ShuffleUp,                InstType.CallQuaternary, HelperFunctionNames.ShuffleUp);
+            Add(Instruction.ShuffleXor,               InstType.CallQuaternary, HelperFunctionNames.ShuffleXor);
             Add(Instruction.Sine,                     InstType.CallUnary,      "sin");
             Add(Instruction.SquareRoot,               InstType.CallUnary,      "sqrt");
             Add(Instruction.StoreLocal,               InstType.Special);

Ryujinx.Graphics.Shader/Instructions/InstEmitMove.cs

@@ -118,25 +118,17 @@ namespace Ryujinx.Graphics.Shader.Instructions
             Operand srcB = op.IsBImmediate ? Const(op.ImmediateB) : Register(op.Rb);
             Operand srcC = op.IsCImmediate ? Const(op.ImmediateC) : Register(op.Rc);
 
-            Operand res = null;
+            (Operand res, Operand valid) = op.ShuffleType switch
-
-            switch (op.ShuffleType)
             {
-                case ShuffleType.Indexed:
+                ShuffleType.Indexed   => context.Shuffle(srcA, srcB, srcC),
-                    res = context.Shuffle(srcA, srcB, srcC);
+                ShuffleType.Up        => context.ShuffleUp(srcA, srcB, srcC),
-                    break;
+                ShuffleType.Down      => context.ShuffleDown(srcA, srcB, srcC),
-                case ShuffleType.Up:
+                ShuffleType.Butterfly => context.ShuffleXor(srcA, srcB, srcC),
-                    res = context.ShuffleUp(srcA, srcB, srcC);
+                _                     => (null, null)
-                    break;
+            };
-                case ShuffleType.Down:
-                    res = context.ShuffleDown(srcA, srcB, srcC);
-                    break;
-                case ShuffleType.Butterfly:
-                    res = context.ShuffleXor(srcA, srcB, srcC);
-                    break;
-            }
 
             context.Copy(GetDest(context), res);
+            context.Copy(pred, valid);
         }
     }
 }

Ryujinx.Graphics.Shader/IntermediateRepresentation/INode.cs

@@ -4,8 +4,10 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
     {
         Operand Dest { get; set; }
 
+        int DestsCount { get; }
         int SourcesCount { get; }
 
+        Operand GetDest(int index);
         Operand GetSource(int index);
 
         void SetSource(int index, Operand operand);

Ryujinx.Graphics.Shader/IntermediateRepresentation/Operation.cs

@@ -6,13 +6,33 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
     {
         public Instruction Inst { get; private set; }
 
-        private Operand _dest;
+        private Operand[] _dests;
 
         public Operand Dest
         {
-            get => _dest;
+            get
-            set => _dest = AssignDest(value);
+            {
+                return _dests.Length != 0 ? _dests[0] : null;
             }
+            set
+            {
+                if (value != null && value.Type == OperandType.LocalVariable)
+                {
+                    value.AsgOp = this;
+                }
+
+                if (value != null)
+                {
+                    _dests = new[] { value };
+                }
+                else
+                {
+                    _dests = Array.Empty<Operand>();
+                }
+            }
+        }
+
+        public int DestsCount => _dests.Length;
 
         private Operand[] _sources;
 
@@ -20,11 +40,8 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
 
         public int Index { get; }
 
-        public Operation(Instruction inst, Operand dest, params Operand[] sources)
+        private Operation(Operand[] sources)
         {
-            Inst = inst;
-            Dest = dest;
-
             // The array may be modified externally, so we store a copy.
             _sources = (Operand[])sources.Clone();
 
@@ -39,11 +56,42 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
             }
         }
 
-        public Operation(
+        public Operation(Instruction inst, int index, Operand[] dests, Operand[] sources) : this(sources)
-            Instruction      inst,
+        {
-            int              index,
+            Inst  = inst;
-            Operand          dest,
+            Index = index;
-            params Operand[] sources) : this(inst, dest, sources)
+
+            // The array may be modified externally, so we store a copy.
+            _dests = (Operand[])dests.Clone();
+
+            for (int dstIndex = 0; dstIndex < dests.Length; dstIndex++)
+            {
+                Operand dest = dests[dstIndex];
+
+                if (dest != null && dest.Type == OperandType.LocalVariable)
+                {
+                    dest.AsgOp = this;
+                }
+            }
+        }
+
+        public Operation(Instruction inst, Operand dest, params Operand[] sources) : this(sources)
+        {
+            Inst = inst;
+
+            if (dest != null)
+            {
+                dest.AsgOp = this;
+
+                _dests = new[] { dest };
+            }
+            else
+            {
+                _dests = Array.Empty<Operand>();
+            }
+        }
+
+        public Operation(Instruction inst, int index, Operand dest, params Operand[] sources) : this(inst, dest, sources)
         {
             Index = index;
         }
@@ -67,14 +115,9 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
             }
         }
 
-        private Operand AssignDest(Operand dest)
+        public Operand GetDest(int index)
         {
-            if (dest != null && dest.Type == OperandType.LocalVariable)
+            return _dests[index];
-            {
-                dest.AsgOp = this;
-            }
-
-            return dest;
         }
 
         public Operand GetSource(int index)
@@ -82,6 +125,23 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
             return _sources[index];
         }
 
+        public void SetDest(int index, Operand dest)
+        {
+            Operand oldDest = _dests[index];
+
+            if (oldDest != null && oldDest.Type == OperandType.LocalVariable)
+            {
+                oldDest.AsgOp = null;
+            }
+
+            if (dest != null && dest.Type == OperandType.LocalVariable)
+            {
+                dest.AsgOp = this;
+            }
+
+            _dests[index] = dest;
+        }
+
         public void SetSource(int index, Operand source)
         {
             Operand oldSrc = _sources[index];

Ryujinx.Graphics.Shader/IntermediateRepresentation/PhiNode.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 
 namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
@@ -12,6 +13,8 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
             set => _dest = AssignDest(value);
         }
 
+        public int DestsCount => _dest != null ? 1 : 0;
+
         private HashSet<BasicBlock> _blocks;
 
         private class PhiSource
@@ -64,6 +67,16 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
             }
         }
 
+        public Operand GetDest(int index)
+        {
+            if (index != 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(index));
+            }
+
+            return _dest;
+        }
+
         public Operand GetSource(int index)
         {
             return _sources[index].Operand;

Ryujinx.Graphics.Shader/StructuredIr/InstructionInfo.cs

@@ -94,13 +94,6 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
             Add(Instruction.LogicalExclusiveOr,       VariableType.Bool,   VariableType.Bool,   VariableType.Bool);
             Add(Instruction.LogicalNot,               VariableType.Bool,   VariableType.Bool);
             Add(Instruction.LogicalOr,                VariableType.Bool,   VariableType.Bool,   VariableType.Bool);
-            Add(Instruction.ShiftLeft,                VariableType.Int,    VariableType.Int,    VariableType.Int);
-            Add(Instruction.ShiftRightS32,            VariableType.S32,    VariableType.S32,    VariableType.Int);
-            Add(Instruction.ShiftRightU32,            VariableType.U32,    VariableType.U32,    VariableType.Int);
-            Add(Instruction.Shuffle,                  VariableType.F32,    VariableType.F32,    VariableType.U32,    VariableType.U32);
-            Add(Instruction.ShuffleDown,              VariableType.F32,    VariableType.F32,    VariableType.U32,    VariableType.U32);
-            Add(Instruction.ShuffleUp,                VariableType.F32,    VariableType.F32,    VariableType.U32,    VariableType.U32);
-            Add(Instruction.ShuffleXor,               VariableType.F32,    VariableType.F32,    VariableType.U32,    VariableType.U32);
             Add(Instruction.Maximum,                  VariableType.Scalar, VariableType.Scalar, VariableType.Scalar);
             Add(Instruction.MaximumU32,               VariableType.U32,    VariableType.U32,    VariableType.U32);
             Add(Instruction.Minimum,                  VariableType.Scalar, VariableType.Scalar, VariableType.Scalar);
@@ -113,6 +106,13 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
             Add(Instruction.PackHalf2x16,             VariableType.U32,    VariableType.F32,    VariableType.F32);
             Add(Instruction.ReciprocalSquareRoot,     VariableType.Scalar, VariableType.Scalar);
             Add(Instruction.Round,                    VariableType.Scalar, VariableType.Scalar);
+            Add(Instruction.ShiftLeft,                VariableType.Int,    VariableType.Int,    VariableType.Int);
+            Add(Instruction.ShiftRightS32,            VariableType.S32,    VariableType.S32,    VariableType.Int);
+            Add(Instruction.ShiftRightU32,            VariableType.U32,    VariableType.U32,    VariableType.Int);
+            Add(Instruction.Shuffle,                  VariableType.F32,    VariableType.F32,    VariableType.U32,    VariableType.U32,    VariableType.Bool);
+            Add(Instruction.ShuffleDown,              VariableType.F32,    VariableType.F32,    VariableType.U32,    VariableType.U32,    VariableType.Bool);
+            Add(Instruction.ShuffleUp,                VariableType.F32,    VariableType.F32,    VariableType.U32,    VariableType.U32,    VariableType.Bool);
+            Add(Instruction.ShuffleXor,               VariableType.F32,    VariableType.F32,    VariableType.U32,    VariableType.U32,    VariableType.Bool);
             Add(Instruction.Sine,                     VariableType.Scalar, VariableType.Scalar);
             Add(Instruction.SquareRoot,               VariableType.Scalar, VariableType.Scalar);
             Add(Instruction.StoreGlobal,              VariableType.None,   VariableType.S32,    VariableType.S32,    VariableType.U32);

Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs

@@ -77,6 +77,7 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
             bool isCall = inst == Instruction.Call;
 
             int sourcesCount = operation.SourcesCount;
+            int outDestsCount = operation.DestsCount != 0 ? operation.DestsCount - 1 : 0;
 
             List<Operand> callOutOperands = new List<Operand>();
 
@@ -93,7 +94,7 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
                 sourcesCount += callOutOperands.Count;
             }
 
-            IAstNode[] sources = new IAstNode[sourcesCount];
+            IAstNode[] sources = new IAstNode[sourcesCount + outDestsCount];
 
             for (int index = 0; index < operation.SourcesCount; index++)
             {
@@ -110,6 +111,15 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
                 callOutOperands.Clear();
             }
 
+            for (int index = 0; index < outDestsCount; index++)
+            {
+                AstOperand oper = context.GetOperandDef(operation.GetDest(1 + index));
+
+                oper.VarType = InstructionInfo.GetSrcVarType(inst, sourcesCount + index);
+
+                sources[sourcesCount + index] = oper;
+            }
+
             AstTextureOperation GetAstTextureOperation(TextureOperation texOp)
             {
                 return new AstTextureOperation(

Ryujinx.Graphics.Shader/Translation/EmitterContext.cs

@@ -37,6 +37,17 @@ namespace Ryujinx.Graphics.Shader.Translation
             return dest;
         }
 
+        public (Operand, Operand) Add(Instruction inst, (Operand, Operand) dest, params Operand[] sources)
+        {
+            Operand[] dests = new[] { dest.Item1, dest.Item2 };
+
+            Operation operation = new Operation(inst, 0, dests, sources);
+
+            Add(operation);
+
+            return dest;
+        }
+
         public void Add(Operation operation)
         {
             _operations.Add(operation);

Ryujinx.Graphics.Shader/Translation/EmitterContextInsts.cs

@@ -588,24 +588,24 @@ namespace Ryujinx.Graphics.Shader.Translation
             return context.Add(Instruction.ShiftRightU32, Local(), a, b);
         }
 
-        public static Operand Shuffle(this EmitterContext context, Operand a, Operand b, Operand c)
+        public static (Operand, Operand) Shuffle(this EmitterContext context, Operand a, Operand b, Operand c)
         {
-            return context.Add(Instruction.Shuffle, Local(), a, b, c);
+            return context.Add(Instruction.Shuffle, (Local(), Local()), a, b, c);
         }
 
-        public static Operand ShuffleDown(this EmitterContext context, Operand a, Operand b, Operand c)
+        public static (Operand, Operand) ShuffleDown(this EmitterContext context, Operand a, Operand b, Operand c)
         {
-            return context.Add(Instruction.ShuffleDown, Local(), a, b, c);
+            return context.Add(Instruction.ShuffleDown, (Local(), Local()), a, b, c);
         }
 
-        public static Operand ShuffleUp(this EmitterContext context, Operand a, Operand b, Operand c)
+        public static (Operand, Operand) ShuffleUp(this EmitterContext context, Operand a, Operand b, Operand c)
         {
-            return context.Add(Instruction.ShuffleUp, Local(), a, b, c);
+            return context.Add(Instruction.ShuffleUp, (Local(), Local()), a, b, c);
         }
 
-        public static Operand ShuffleXor(this EmitterContext context, Operand a, Operand b, Operand c)
+        public static (Operand, Operand) ShuffleXor(this EmitterContext context, Operand a, Operand b, Operand c)
         {
-            return context.Add(Instruction.ShuffleXor, Local(), a, b, c);
+            return context.Add(Instruction.ShuffleXor, (Local(), Local()), a, b, c);
         }
 
         public static Operand StoreGlobal(this EmitterContext context, Operand a, Operand b, Operand c)

Ryujinx.Graphics.Shader/Translation/Optimizations/Optimizer.cs

@@ -64,7 +64,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
                             else if ((operation.Inst == Instruction.PackHalf2x16 && PropagatePack(operation)) ||
                                      (operation.Inst == Instruction.ShuffleXor   && MatchDdxOrDdy(operation)))
                             {
-                                if (operation.Dest.UseOps.Count == 0)
+                                if (DestHasNoUses(operation))
                                 {
                                     RemoveNode(block, node);
                                 }
@@ -260,6 +260,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
 
                     if (src.UseOps.Remove(node) && src.UseOps.Count == 0)
                     {
+                        Debug.Assert(src.AsgOp != null);
                         nodes.Enqueue(src.AsgOp);
                     }
                 }
@@ -268,7 +269,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
 
         private static bool IsUnused(INode node)
         {
-            return !HasSideEffects(node) && DestIsLocalVar(node) && node.Dest.UseOps.Count == 0;
+            return !HasSideEffects(node) && DestIsLocalVar(node) && DestHasNoUses(node);
         }
 
         private static bool HasSideEffects(INode node)
@@ -298,7 +299,33 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
 
         private static bool DestIsLocalVar(INode node)
         {
-            return node.Dest != null && node.Dest.Type == OperandType.LocalVariable;
+            if (node.DestsCount == 0)
+            {
+                return false;
+            }
+
+            for (int index = 0; index < node.DestsCount; index++)
+            {
+                if (node.GetDest(index).Type != OperandType.LocalVariable)
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        private static bool DestHasNoUses(INode node)
+        {
+            for (int index = 0; index < node.DestsCount; index++)
+            {
+                if (node.GetDest(index).UseOps.Count != 0)
+                {
+                    return false;
+                }
+            }
+
+            return true;
         }
     }
 }

Ryujinx.Graphics.Shader/Translation/Ssa.cs

@@ -116,13 +116,18 @@ namespace Ryujinx.Graphics.Shader.Translation
                             operation.SetSource(index, RenameLocal(operation.GetSource(index)));
                         }
 
-                        if (operation.Dest != null && operation.Dest.Type == OperandType.Register)
+                        for (int index = 0; index < operation.DestsCount; index++)
+                        {
+                            Operand dest = operation.GetDest(index);
+
+                            if (dest.Type == OperandType.Register)
                             {
                                 Operand local = Local();
 
-                            localDefs[GetKeyFromRegister(operation.Dest.GetRegister())] = local;
+                                localDefs[GetKeyFromRegister(dest.GetRegister())] = local;
 
-                            operation.Dest = local;
+                                operation.SetDest(index, local);
+                            }
                         }
                     }
 
@@ -185,9 +190,7 @@ namespace Ryujinx.Graphics.Shader.Translation
                     return operand;
                 }
 
-                LinkedListNode<INode> node = block.Operations.First;
+                for (LinkedListNode<INode> node = block.Operations.First; node != null; node = node.Next)
-
-                while (node != null)
                 {
                     if (node.Value is Operation operation)
                     {
@@ -196,8 +199,6 @@ namespace Ryujinx.Graphics.Shader.Translation
                             operation.SetSource(index, RenameGlobal(operation.GetSource(index)));
                         }
                     }
-
-                    node = node.Next;
                 }
             }
         }