ARMeilleure: Add gfni acceleration (#3669)

* ARMeilleure: Add `GFNI` detection

This is intended for utilizing the `gf2p8affineqb` instruction

* ARMeilleure: Add `gf2p8affineqb`

Not using the VEX or EVEX-form of this instruction is intentional. There
are `GFNI`-chips that do not support AVX(so no VEX encoding) such as
Tremont(Lakefield) chips as well as Jasper Lake.

13df339fe7/GenuineIntel/GenuineIntel00806A1_Lakefield_LC_InstLatX64.txt (L1297-L1299)

13df339fe7/GenuineIntel/GenuineIntel00906C0_JasperLake_InstLatX64.txt (L1252-L1254)

* ARMeilleure: Add `gfni` acceleration of `Rbit_V`

Passes all `Rbit_V*` unit tests on my `i9-11900k`

* ARMeilleure: Add `gfni` acceleration of `S{l,r}i_V`

Also added a fast-path for when the shift amount is greater than the
size of the element.

* ARMeilleure: Add `gfni` acceleration of `Shl_V` and `Sshr_V`

* ARMeilleure: Increment InternalVersion

* ARMeilleure: Fix Intrinsic and Assembler Table alignment

`gf2p8affineqb` is the longest instruction name I know of. It shouldn't
get any wider than this.

* ARMeilleure: Remove SSE2+SHA requirement for GFNI

* ARMeilleure Add `X86GetGf2p8LogicalShiftLeft`

Used to generate GF(2^8) 8x8 bit-matrices for bit-shifting for the `gf2p8affineqb` instruction.

* ARMeilleure: Append `FeatureInfo7Ecx` to `FeatureInfo`

ARMeilleure/CodeGen/X86/AssemblerTable.cs

@@ -113,6 +113,7 @@ namespace ARMeilleure.CodeGen.X86
             Add(X86Instruction.Divps,         new InstructionInfo(BadOp,      BadOp,      BadOp,      BadOp,      0x00000f5e, InstructionFlags.Vex));
             Add(X86Instruction.Divsd,         new InstructionInfo(BadOp,      BadOp,      BadOp,      BadOp,      0x00000f5e, InstructionFlags.Vex | InstructionFlags.PrefixF2));
             Add(X86Instruction.Divss,         new InstructionInfo(BadOp,      BadOp,      BadOp,      BadOp,      0x00000f5e, InstructionFlags.Vex | InstructionFlags.PrefixF3));
+            Add(X86Instruction.Gf2p8affineqb, new InstructionInfo(BadOp,      BadOp,      BadOp,      BadOp,      0x000f3ace, InstructionFlags.Prefix66));
             Add(X86Instruction.Haddpd,        new InstructionInfo(BadOp,      BadOp,      BadOp,      BadOp,      0x00000f7c, InstructionFlags.Vex | InstructionFlags.Prefix66));
             Add(X86Instruction.Haddps,        new InstructionInfo(BadOp,      BadOp,      BadOp,      BadOp,      0x00000f7c, InstructionFlags.Vex | InstructionFlags.PrefixF2));
             Add(X86Instruction.Idiv,          new InstructionInfo(BadOp,      BadOp,      BadOp,      BadOp,      0x070000f7, InstructionFlags.None));

ARMeilleure/CodeGen/X86/HardwareCapabilities.cs

@@ -20,8 +20,9 @@ namespace ARMeilleure.CodeGen.X86
 
             if (maxNum >= 7)
             {
-                (_, int ebx7, _, _) = X86Base.CpuId(0x00000007, 0x00000000);
+                (_, int ebx7, int ecx7, _) = X86Base.CpuId(0x00000007, 0x00000000);
                 FeatureInfo7Ebx = (FeatureFlags7Ebx)ebx7;
+                FeatureInfo7Ecx = (FeatureFlags7Ecx)ecx7;
             }
         }
 
@@ -54,9 +55,16 @@ namespace ARMeilleure.CodeGen.X86
             Sha = 1 << 29
         }
 
+        [Flags]
+        public enum FeatureFlags7Ecx
+        {
+            Gfni = 1 << 8,
+        }
+
         public static FeatureFlags1Edx FeatureInfo1Edx { get; }
         public static FeatureFlags1Ecx FeatureInfo1Ecx { get; }
         public static FeatureFlags7Ebx FeatureInfo7Ebx { get; } = 0;
+        public static FeatureFlags7Ecx FeatureInfo7Ecx { get; } = 0;
 
         public static bool SupportsSse => FeatureInfo1Edx.HasFlag(FeatureFlags1Edx.Sse);
         public static bool SupportsSse2 => FeatureInfo1Edx.HasFlag(FeatureFlags1Edx.Sse2);
@@ -72,6 +80,7 @@ namespace ARMeilleure.CodeGen.X86
         public static bool SupportsAvx2 => FeatureInfo7Ebx.HasFlag(FeatureFlags7Ebx.Avx2) && SupportsAvx;
         public static bool SupportsF16c => FeatureInfo1Ecx.HasFlag(FeatureFlags1Ecx.F16c);
         public static bool SupportsSha => FeatureInfo7Ebx.HasFlag(FeatureFlags7Ebx.Sha);
+        public static bool SupportsGfni => FeatureInfo7Ecx.HasFlag(FeatureFlags7Ecx.Gfni);
 
         public static bool ForceLegacySse { get; set; }
 

ARMeilleure/CodeGen/X86/IntrinsicTable.cs

@@ -58,6 +58,7 @@ namespace ARMeilleure.CodeGen.X86
             Add(Intrinsic.X86Divps,         new IntrinsicInfo(X86Instruction.Divps,         IntrinsicType.Binary));
             Add(Intrinsic.X86Divsd,         new IntrinsicInfo(X86Instruction.Divsd,         IntrinsicType.Binary));
             Add(Intrinsic.X86Divss,         new IntrinsicInfo(X86Instruction.Divss,         IntrinsicType.Binary));
+            Add(Intrinsic.X86Gf2p8affineqb, new IntrinsicInfo(X86Instruction.Gf2p8affineqb, IntrinsicType.TernaryImm));
             Add(Intrinsic.X86Haddpd,        new IntrinsicInfo(X86Instruction.Haddpd,        IntrinsicType.Binary));
             Add(Intrinsic.X86Haddps,        new IntrinsicInfo(X86Instruction.Haddps,        IntrinsicType.Binary));
             Add(Intrinsic.X86Insertps,      new IntrinsicInfo(X86Instruction.Insertps,      IntrinsicType.TernaryImm));

ARMeilleure/CodeGen/X86/X86Instruction.cs

@@ -54,6 +54,7 @@ namespace ARMeilleure.CodeGen.X86
         Divps,
         Divsd,
         Divss,
+        Gf2p8affineqb,
         Haddpd,
         Haddps,
         Idiv,

ARMeilleure/Instructions/InstEmitSimdHelper.cs

@@ -243,6 +243,21 @@ namespace ARMeilleure.Instructions
             throw new ArgumentException($"Invalid rounding mode \"{roundMode}\".");
         }
 
+        public static ulong X86GetGf2p8LogicalShiftLeft(int shift)
+        {
+            ulong identity = 
+                (0b00000001UL << 56) |
+                (0b00000010UL << 48) |
+                (0b00000100UL << 40) |
+                (0b00001000UL << 32) |
+                (0b00010000UL << 24) |
+                (0b00100000UL << 16) |
+                (0b01000000UL <<  8) |
+                (0b10000000UL <<  0);
+
+            return shift >= 0 ? identity >> (shift * 8) : identity << (-shift * 8);
+        }
+
         public static Operand EmitCountSetBits8(ArmEmitterContext context, Operand op) // "size" is 8 (SIMD&FP Inst.).
         {
             Debug.Assert(op.Type == OperandType.I32 || op.Type == OperandType.I64);

ARMeilleure/Instructions/InstEmitSimdLogical.cs

@@ -336,8 +336,32 @@ namespace ARMeilleure.Instructions
         {
             OpCodeSimd op = (OpCodeSimd)context.CurrOp;
 
-            Operand res = context.VectorZero();
+            if (Optimizations.UseGfni)
+            {
+                const long bitMatrix =
+                    (0b10000000L << 56) |
+                    (0b01000000L << 48) |
+                    (0b00100000L << 40) |
+                    (0b00010000L << 32) |
+                    (0b00001000L << 24) |
+                    (0b00000100L << 16) |
+                    (0b00000010L <<  8) |
+                    (0b00000001L <<  0);
 
+                Operand vBitMatrix = X86GetAllElements(context, bitMatrix);
+
+                Operand res = context.AddIntrinsic(Intrinsic.X86Gf2p8affineqb, GetVec(op.Rn), vBitMatrix, Const(0));
+
+                if (op.RegisterSize == RegisterSize.Simd64)
+                {
+                    res = context.VectorZeroUpper64(res);
+                }
+
+                context.Copy(GetVec(op.Rd), res);
+            }
+            else
+            {
+                Operand res = context.VectorZero();
                 int elems = op.RegisterSize == RegisterSize.Simd128 ? 16 : 8;
 
                 for (int index = 0; index < elems; index++)
@@ -351,6 +375,7 @@ namespace ARMeilleure.Instructions
 
                 context.Copy(GetVec(op.Rd), res);
             }
+        }
 
         private static Operand EmitReverseBits8Op(ArmEmitterContext context, Operand op)
         {

ARMeilleure/Instructions/InstEmitSimdShift.cs

@@ -88,8 +88,35 @@ namespace ARMeilleure.Instructions
             OpCodeSimdShImm op = (OpCodeSimdShImm)context.CurrOp;
 
             int shift = GetImmShl(op);
+            int eSize = 8 << op.Size;
 
-            if (Optimizations.UseSse2 && op.Size > 0)
+            if (shift >= eSize)
+            {
+                if ((op.RegisterSize == RegisterSize.Simd64))
+                {
+                    Operand res = context.VectorZeroUpper64(GetVec(op.Rd));
+
+                    context.Copy(GetVec(op.Rd), res);
+                }
+            }
+            else if (Optimizations.UseGfni && op.Size == 0)
+            {
+                Operand n = GetVec(op.Rn);
+
+                ulong bitMatrix = X86GetGf2p8LogicalShiftLeft(shift);
+
+                Operand vBitMatrix = X86GetElements(context, bitMatrix, bitMatrix);
+
+                Operand res = context.AddIntrinsic(Intrinsic.X86Gf2p8affineqb, n, vBitMatrix, Const(0));
+
+                if (op.RegisterSize == RegisterSize.Simd64)
+                {
+                    res = context.VectorZeroUpper64(res);
+                }
+
+                context.Copy(GetVec(op.Rd), res);
+            }
+            else if (Optimizations.UseSse2 && op.Size > 0)
             {
                 Operand n = GetVec(op.Rn);
 
@@ -396,10 +423,40 @@ namespace ARMeilleure.Instructions
         {
             OpCodeSimdShImm op = (OpCodeSimdShImm)context.CurrOp;
 
-            if (Optimizations.UseSse2 && op.Size > 0 && op.Size < 3)
-            {
             int shift = GetImmShr(op);
 
+            if (Optimizations.UseGfni && op.Size == 0)
+            {
+                Operand n = GetVec(op.Rn);
+
+                ulong bitMatrix;
+
+                if (shift < 8)
+                {
+                    bitMatrix = X86GetGf2p8LogicalShiftLeft(-shift);
+
+                    // Extend sign-bit
+                    bitMatrix |= 0x8080808080808080UL >> (64 - shift * 8);
+                }
+                else
+                {
+                    // Replicate sign-bit into all bits
+                    bitMatrix = 0x8080808080808080UL;
+                }
+
+                Operand vBitMatrix = X86GetElements(context, bitMatrix, bitMatrix);
+
+                Operand res = context.AddIntrinsic(Intrinsic.X86Gf2p8affineqb, n, vBitMatrix, Const(0));
+
+                if (op.RegisterSize == RegisterSize.Simd64)
+                {
+                    res = context.VectorZeroUpper64(res);
+                }
+
+                context.Copy(GetVec(op.Rd), res);
+            }
+            else if (Optimizations.UseSse2 && op.Size > 0 && op.Size < 3)
+            {
                 Operand n = GetVec(op.Rn);
 
                 Intrinsic sraInst = X86PsraInstruction[op.Size];
@@ -929,10 +986,44 @@ namespace ARMeilleure.Instructions
             OpCodeSimdShImm op = (OpCodeSimdShImm)context.CurrOp;
 
             int shift = GetImmShl(op);
+            int eSize = 8 << op.Size;
 
             ulong mask = shift != 0 ? ulong.MaxValue >> (64 - shift) : 0UL;
 
-            if (Optimizations.UseSse2 && op.Size > 0)
+            if (shift >= eSize)
+            {
+                if ((op.RegisterSize == RegisterSize.Simd64) || scalar)
+                {
+                    Operand res = context.VectorZeroUpper64(GetVec(op.Rd));
+
+                    context.Copy(GetVec(op.Rd), res);
+                }
+            }
+            else if (Optimizations.UseGfni && op.Size == 0)
+            {
+                Operand d = GetVec(op.Rd);
+                Operand n = GetVec(op.Rn);
+
+                ulong bitMatrix = X86GetGf2p8LogicalShiftLeft(shift);
+
+                Operand vBitMatrix = X86GetElements(context, bitMatrix, bitMatrix);
+
+                Operand nShifted = context.AddIntrinsic(Intrinsic.X86Gf2p8affineqb, n, vBitMatrix, Const(0));
+
+                Operand dMask = X86GetAllElements(context, (long)mask * _masks_SliSri[op.Size]);
+
+                Operand dMasked = context.AddIntrinsic(Intrinsic.X86Pand, d, dMask);
+
+                Operand res = context.AddIntrinsic(Intrinsic.X86Por, nShifted, dMasked);
+
+                if ((op.RegisterSize == RegisterSize.Simd64) || scalar)
+                {
+                    res = context.VectorZeroUpper64(res);
+                }
+
+                context.Copy(d, res);
+            }
+            else if (Optimizations.UseSse2 && op.Size > 0)
             {
                 Operand d = GetVec(op.Rd);
                 Operand n = GetVec(op.Rn);
@@ -988,7 +1079,40 @@ namespace ARMeilleure.Instructions
 
             ulong mask = (ulong.MaxValue << (eSize - shift)) & (ulong.MaxValue >> (64 - eSize));
 
-            if (Optimizations.UseSse2 && op.Size > 0)
+            if (shift >= eSize)
+            {
+                if ((op.RegisterSize == RegisterSize.Simd64) || scalar)
+                {
+                    Operand res = context.VectorZeroUpper64(GetVec(op.Rd));
+
+                    context.Copy(GetVec(op.Rd), res);
+                }
+            }
+            else if (Optimizations.UseGfni && op.Size == 0)
+            {
+                Operand d = GetVec(op.Rd);
+                Operand n = GetVec(op.Rn);
+
+                ulong bitMatrix = X86GetGf2p8LogicalShiftLeft(-shift);
+
+                Operand vBitMatrix = X86GetElements(context, bitMatrix, bitMatrix);
+
+                Operand nShifted = context.AddIntrinsic(Intrinsic.X86Gf2p8affineqb, n, vBitMatrix, Const(0));
+
+                Operand dMask = X86GetAllElements(context, (long)mask * _masks_SliSri[op.Size]);
+
+                Operand dMasked = context.AddIntrinsic(Intrinsic.X86Pand, d, dMask);
+
+                Operand res = context.AddIntrinsic(Intrinsic.X86Por, nShifted, dMasked);
+
+                if ((op.RegisterSize == RegisterSize.Simd64) || scalar)
+                {
+                    res = context.VectorZeroUpper64(res);
+                }
+
+                context.Copy(d, res);
+            }
+            else if (Optimizations.UseSse2 && op.Size > 0)
             {
                 Operand d = GetVec(op.Rd);
                 Operand n = GetVec(op.Rn);

ARMeilleure/IntermediateRepresentation/Intrinsic.cs

@@ -47,6 +47,7 @@ namespace ARMeilleure.IntermediateRepresentation
         X86Divps,
         X86Divsd,
         X86Divss,
+        X86Gf2p8affineqb,
         X86Haddpd,
         X86Haddps,
         X86Insertps,

ARMeilleure/Optimizations.cs

@@ -22,6 +22,7 @@ namespace ARMeilleure
         public static bool UseAesniIfAvailable     { get; set; } = true;
         public static bool UsePclmulqdqIfAvailable { get; set; } = true;
         public static bool UseShaIfAvailable       { get; set; } = true;
+        public static bool UseGfniIfAvailable      { get; set; } = true;
 
         public static bool ForceLegacySse
         {
@@ -42,5 +43,6 @@ namespace ARMeilleure
         internal static bool UseAesni     => UseAesniIfAvailable     && HardwareCapabilities.SupportsAesni;
         internal static bool UsePclmulqdq => UsePclmulqdqIfAvailable && HardwareCapabilities.SupportsPclmulqdq;
         internal static bool UseSha       => UseShaIfAvailable       && HardwareCapabilities.SupportsSha;
+        internal static bool UseGfni      => UseGfniIfAvailable      && HardwareCapabilities.SupportsGfni;
     }
 }

ARMeilleure/Translation/PTC/Ptc.cs

@@ -27,7 +27,7 @@ namespace ARMeilleure.Translation.PTC
         private const string OuterHeaderMagicString = "PTCohd\0\0";
         private const string InnerHeaderMagicString = "PTCihd\0\0";
 
-        private const uint InternalVersion = 3703; //! To be incremented manually for each change to the ARMeilleure project.
+        private const uint InternalVersion = 3710; //! To be incremented manually for each change to the ARMeilleure project.
 
         private const string ActualDir = "0";
         private const string BackupDir = "1";
@@ -951,7 +951,8 @@ namespace ARMeilleure.Translation.PTC
             return new FeatureInfo(
                 (uint)HardwareCapabilities.FeatureInfo1Ecx,
                 (uint)HardwareCapabilities.FeatureInfo1Edx,
-                (uint)HardwareCapabilities.FeatureInfo7Ebx);
+                (uint)HardwareCapabilities.FeatureInfo7Ebx,
+                (uint)HardwareCapabilities.FeatureInfo7Ecx);
         }
 
         private static byte GetMemoryManagerMode()
@@ -971,7 +972,7 @@ namespace ARMeilleure.Translation.PTC
             return osPlatform;
         }
 
-        [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 54*/)]
+        [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 58*/)]
         private struct OuterHeader
         {
             public ulong Magic;
@@ -1002,8 +1003,8 @@ namespace ARMeilleure.Translation.PTC
             }
         }
 
-        [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 12*/)]
+        [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 16*/)]
-        private record struct FeatureInfo(uint FeatureInfo0, uint FeatureInfo1, uint FeatureInfo2);
+        private record struct FeatureInfo(uint FeatureInfo0, uint FeatureInfo1, uint FeatureInfo2, uint FeatureInfo3);
 
         [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 128*/)]
         private struct InnerHeader