From 0915731a9dfc4e2b9263d4b30c2876446ff2d9b3 Mon Sep 17 00:00:00 2001 From: LDj3SNuD <35856442+LDj3SNuD@users.noreply.github.com> Date: Mon, 30 Dec 2019 02:22:47 +0100 Subject: [PATCH] Implemented fast paths for: (#846) * opt * Nit. * opt_p2 * Nit. --- ARMeilleure/CodeGen/X86/Assembler.cs | 11 + ARMeilleure/CodeGen/X86/CodeGenerator.cs | 28 +- ARMeilleure/CodeGen/X86/IntrinsicTable.cs | 3 + ARMeilleure/CodeGen/X86/IntrinsicType.cs | 1 + ARMeilleure/CodeGen/X86/X86Instruction.cs | 1 + ARMeilleure/Instructions/InstEmitAlu.cs | 81 +- .../Instructions/InstEmitSimdArithmetic.cs | 87 +- ARMeilleure/Instructions/InstEmitSimdCvt.cs | 751 +++++++++++++----- .../Instructions/InstEmitSimdHelper.cs | 79 +- .../Instructions/InstEmitSimdLogical.cs | 25 +- ARMeilleure/Instructions/InstEmitSimdMove.cs | 24 +- ARMeilleure/Instructions/InstEmitSimdShift.cs | 62 +- ARMeilleure/Instructions/SoftFallback.cs | 69 -- .../IntermediateRepresentation/Intrinsic.cs | 3 + Ryujinx.Tests/Cpu/CpuTestSimd.cs | 4 +- Ryujinx.Tests/Cpu/CpuTestSimdCvt.cs | 4 +- Ryujinx.Tests/Cpu/CpuTestSimdShImm.cs | 2 +- 17 files changed, 856 insertions(+), 379 deletions(-) diff --git a/ARMeilleure/CodeGen/X86/Assembler.cs b/ARMeilleure/CodeGen/X86/Assembler.cs index 24a122c3..4568253a 100644 --- a/ARMeilleure/CodeGen/X86/Assembler.cs +++ b/ARMeilleure/CodeGen/X86/Assembler.cs @@ -103,6 +103,7 @@ namespace ARMeilleure.CodeGen.X86 Add(X86Instruction.Cvtsi2sd, new InstructionInfo(BadOp, BadOp, BadOp, BadOp, 0x00000f2a, InstructionFlags.Vex | InstructionFlags.PrefixF2)); Add(X86Instruction.Cvtsi2ss, new InstructionInfo(BadOp, BadOp, BadOp, BadOp, 0x00000f2a, InstructionFlags.Vex | InstructionFlags.PrefixF3)); Add(X86Instruction.Cvtss2sd, new InstructionInfo(BadOp, BadOp, BadOp, BadOp, 0x00000f5a, InstructionFlags.Vex | InstructionFlags.PrefixF3)); + Add(X86Instruction.Cvtss2si, new InstructionInfo(BadOp, BadOp, BadOp, BadOp, 0x00000f2d, InstructionFlags.Vex | InstructionFlags.PrefixF3)); Add(X86Instruction.Div, new InstructionInfo(BadOp, BadOp, BadOp, BadOp, 0x060000f7, InstructionFlags.None)); Add(X86Instruction.Divpd, new InstructionInfo(BadOp, BadOp, BadOp, BadOp, 0x00000f5e, InstructionFlags.Vex | InstructionFlags.Prefix66)); Add(X86Instruction.Divps, new InstructionInfo(BadOp, BadOp, BadOp, BadOp, 0x00000f5e, InstructionFlags.Vex)); @@ -791,6 +792,16 @@ namespace ARMeilleure.CodeGen.X86 } } + public void WriteInstruction( + X86Instruction inst, + Operand dest, + Operand src1, + Operand src2, + OperandType type) + { + WriteInstruction(dest, src1, src2, inst, type); + } + public void WriteInstruction(X86Instruction inst, Operand dest, Operand source, byte imm) { WriteInstruction(dest, null, source, inst); diff --git a/ARMeilleure/CodeGen/X86/CodeGenerator.cs b/ARMeilleure/CodeGen/X86/CodeGenerator.cs index d1224363..0268665c 100644 --- a/ARMeilleure/CodeGen/X86/CodeGenerator.cs +++ b/ARMeilleure/CodeGen/X86/CodeGenerator.cs @@ -269,11 +269,11 @@ namespace ARMeilleure.CodeGen.X86 { if (dest.Type == OperandType.I32) { - context.Assembler.Movd(dest, source); // int _mm_cvtsi128_si32 + context.Assembler.Movd(dest, source); // int _mm_cvtsi128_si32(__m128i a) } else /* if (dest.Type == OperandType.I64) */ { - context.Assembler.Movq(dest, source); // __int64 _mm_cvtsi128_si64 + context.Assembler.Movq(dest, source); // __int64 _mm_cvtsi128_si64(__m128i a) } } else @@ -305,6 +305,26 @@ namespace ARMeilleure.CodeGen.X86 break; } + case IntrinsicType.BinaryGpr: + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + EnsureSameType(dest, src1); + + if (!HardwareCapabilities.SupportsVexEncoding) + { + EnsureSameReg(dest, src1); + } + + Debug.Assert(!dest.Type.IsInteger() && src2.Type.IsInteger()); + + context.Assembler.WriteInstruction(info.Inst, dest, src1, src2, src2.Type); + + break; + } + case IntrinsicType.BinaryImm: { Operand dest = operation.Destination; @@ -1070,11 +1090,11 @@ namespace ARMeilleure.CodeGen.X86 if (source.Type == OperandType.I32) { - context.Assembler.Movd(dest, source); + context.Assembler.Movd(dest, source); // (__m128i _mm_cvtsi32_si128(int a)) } else /* if (source.Type == OperandType.I64) */ { - context.Assembler.Movq(dest, source); + context.Assembler.Movq(dest, source); // (__m128i _mm_cvtsi64_si128(__int64 a)) } } diff --git a/ARMeilleure/CodeGen/X86/IntrinsicTable.cs b/ARMeilleure/CodeGen/X86/IntrinsicTable.cs index e87de035..fd3b691d 100644 --- a/ARMeilleure/CodeGen/X86/IntrinsicTable.cs +++ b/ARMeilleure/CodeGen/X86/IntrinsicTable.cs @@ -41,8 +41,11 @@ namespace ARMeilleure.CodeGen.X86 Add(Intrinsic.X86Cvtps2pd, new IntrinsicInfo(X86Instruction.Cvtps2pd, IntrinsicType.Unary)); Add(Intrinsic.X86Cvtsd2si, new IntrinsicInfo(X86Instruction.Cvtsd2si, IntrinsicType.UnaryToGpr)); Add(Intrinsic.X86Cvtsd2ss, new IntrinsicInfo(X86Instruction.Cvtsd2ss, IntrinsicType.Binary)); + Add(Intrinsic.X86Cvtsi2sd, new IntrinsicInfo(X86Instruction.Cvtsi2sd, IntrinsicType.BinaryGpr)); Add(Intrinsic.X86Cvtsi2si, new IntrinsicInfo(X86Instruction.Movd, IntrinsicType.UnaryToGpr)); + Add(Intrinsic.X86Cvtsi2ss, new IntrinsicInfo(X86Instruction.Cvtsi2ss, IntrinsicType.BinaryGpr)); Add(Intrinsic.X86Cvtss2sd, new IntrinsicInfo(X86Instruction.Cvtss2sd, IntrinsicType.Binary)); + Add(Intrinsic.X86Cvtss2si, new IntrinsicInfo(X86Instruction.Cvtss2si, IntrinsicType.UnaryToGpr)); Add(Intrinsic.X86Divpd, new IntrinsicInfo(X86Instruction.Divpd, IntrinsicType.Binary)); Add(Intrinsic.X86Divps, new IntrinsicInfo(X86Instruction.Divps, IntrinsicType.Binary)); Add(Intrinsic.X86Divsd, new IntrinsicInfo(X86Instruction.Divsd, IntrinsicType.Binary)); diff --git a/ARMeilleure/CodeGen/X86/IntrinsicType.cs b/ARMeilleure/CodeGen/X86/IntrinsicType.cs index 4e9b33e1..41c52b59 100644 --- a/ARMeilleure/CodeGen/X86/IntrinsicType.cs +++ b/ARMeilleure/CodeGen/X86/IntrinsicType.cs @@ -7,6 +7,7 @@ namespace ARMeilleure.CodeGen.X86 Unary, UnaryToGpr, Binary, + BinaryGpr, BinaryImm, Ternary, TernaryImm diff --git a/ARMeilleure/CodeGen/X86/X86Instruction.cs b/ARMeilleure/CodeGen/X86/X86Instruction.cs index a29e68fb..813730f2 100644 --- a/ARMeilleure/CodeGen/X86/X86Instruction.cs +++ b/ARMeilleure/CodeGen/X86/X86Instruction.cs @@ -38,6 +38,7 @@ namespace ARMeilleure.CodeGen.X86 Cvtsi2sd, Cvtsi2ss, Cvtss2sd, + Cvtss2si, Div, Divpd, Divps, diff --git a/ARMeilleure/Instructions/InstEmitAlu.cs b/ARMeilleure/Instructions/InstEmitAlu.cs index 947c9f70..ed1faae4 100644 --- a/ARMeilleure/Instructions/InstEmitAlu.cs +++ b/ARMeilleure/Instructions/InstEmitAlu.cs @@ -2,6 +2,7 @@ using ARMeilleure.Decoders; using ARMeilleure.IntermediateRepresentation; using ARMeilleure.State; using ARMeilleure.Translation; +using System.Diagnostics; using static ARMeilleure.Instructions.InstEmitAluHelper; using static ARMeilleure.Instructions.InstEmitHelper; @@ -265,16 +266,52 @@ namespace ARMeilleure.Instructions if (op.RegisterSize == RegisterSize.Int32) { - d = context.Call(new _U32_U32(SoftFallback.ReverseBits32), n); + d = EmitReverseBits32Op(context, n); } else { - d = context.Call(new _U64_U64(SoftFallback.ReverseBits64), n); + d = EmitReverseBits64Op(context, n); } SetAluDOrZR(context, d); } + private static Operand EmitReverseBits32Op(ArmEmitterContext context, Operand op) + { + Debug.Assert(op.Type == OperandType.I32); + + Operand val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(op, Const(0xaaaaaaaau)), Const(1)), + context.ShiftLeft (context.BitwiseAnd(op, Const(0x55555555u)), Const(1))); + + val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xccccccccu)), Const(2)), + context.ShiftLeft (context.BitwiseAnd(val, Const(0x33333333u)), Const(2))); + val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xf0f0f0f0u)), Const(4)), + context.ShiftLeft (context.BitwiseAnd(val, Const(0x0f0f0f0fu)), Const(4))); + val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xff00ff00u)), Const(8)), + context.ShiftLeft (context.BitwiseAnd(val, Const(0x00ff00ffu)), Const(8))); + + return context.BitwiseOr(context.ShiftRightUI(val, Const(16)), context.ShiftLeft(val, Const(16))); + } + + private static Operand EmitReverseBits64Op(ArmEmitterContext context, Operand op) + { + Debug.Assert(op.Type == OperandType.I64); + + Operand val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(op, Const(0xaaaaaaaaaaaaaaaaul)), Const(1)), + context.ShiftLeft (context.BitwiseAnd(op, Const(0x5555555555555555ul)), Const(1))); + + val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xccccccccccccccccul)), Const(2)), + context.ShiftLeft (context.BitwiseAnd(val, Const(0x3333333333333333ul)), Const(2))); + val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xf0f0f0f0f0f0f0f0ul)), Const(4)), + context.ShiftLeft (context.BitwiseAnd(val, Const(0x0f0f0f0f0f0f0f0ful)), Const(4))); + val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xff00ff00ff00ff00ul)), Const(8)), + context.ShiftLeft (context.BitwiseAnd(val, Const(0x00ff00ff00ff00fful)), Const(8))); + val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xffff0000ffff0000ul)), Const(16)), + context.ShiftLeft (context.BitwiseAnd(val, Const(0x0000ffff0000fffful)), Const(16))); + + return context.BitwiseOr(context.ShiftRightUI(val, Const(32)), context.ShiftLeft(val, Const(32))); + } + public static void Rev16(ArmEmitterContext context) { OpCodeAlu op = (OpCodeAlu)context.CurrOp; @@ -284,32 +321,60 @@ namespace ARMeilleure.Instructions if (op.RegisterSize == RegisterSize.Int32) { - d = context.Call(new _U32_U32(SoftFallback.ReverseBytes16_32), n); + d = EmitReverseBytes16_32Op(context, n); } else { - d = context.Call(new _U64_U64(SoftFallback.ReverseBytes16_64), n); + d = EmitReverseBytes16_64Op(context, n); } SetAluDOrZR(context, d); } + private static Operand EmitReverseBytes16_32Op(ArmEmitterContext context, Operand op) + { + Debug.Assert(op.Type == OperandType.I32); + + Operand val = EmitReverseBytes16_64Op(context, context.ZeroExtend32(OperandType.I64, op)); + + return context.ConvertI64ToI32(val); + } + + private static Operand EmitReverseBytes16_64Op(ArmEmitterContext context, Operand op) + { + Debug.Assert(op.Type == OperandType.I64); + + return context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(op, Const(0xff00ff00ff00ff00ul)), Const(8)), + context.ShiftLeft (context.BitwiseAnd(op, Const(0x00ff00ff00ff00fful)), Const(8))); + } + public static void Rev32(ArmEmitterContext context) { OpCodeAlu op = (OpCodeAlu)context.CurrOp; Operand n = GetIntOrZR(context, op.Rn); + Operand d; if (op.RegisterSize == RegisterSize.Int32) { - SetAluDOrZR(context, context.ByteSwap(n)); + d = context.ByteSwap(n); } else { - Operand d = context.Call(new _U64_U64(SoftFallback.ReverseBytes32_64), n); - - SetAluDOrZR(context, d); + d = EmitReverseBytes32_64Op(context, n); } + + SetAluDOrZR(context, d); + } + + private static Operand EmitReverseBytes32_64Op(ArmEmitterContext context, Operand op) + { + Debug.Assert(op.Type == OperandType.I64); + + Operand val = EmitReverseBytes16_64Op(context, op); + + return context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xffff0000ffff0000ul)), Const(16)), + context.ShiftLeft (context.BitwiseAnd(val, Const(0x0000ffff0000fffful)), Const(16))); } public static void Rev64(ArmEmitterContext context) diff --git a/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs b/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs index 1a9e01c8..4603ae0b 100644 --- a/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs +++ b/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs @@ -77,7 +77,14 @@ namespace ARMeilleure.Instructions public static void Addp_V(ArmEmitterContext context) { - EmitVectorPairwiseOpZx(context, (op1, op2) => context.Add(op1, op2)); + if (Optimizations.UseSsse3) + { + EmitSsse3VectorPairwiseOp(context, X86PaddInstruction); + } + else + { + EmitVectorPairwiseOpZx(context, (op1, op2) => context.Add(op1, op2)); + } } public static void Addv_V(ArmEmitterContext context) @@ -399,7 +406,7 @@ namespace ARMeilleure.Instructions { if (Optimizations.FastFP && Optimizations.UseSse2) { - EmitVectorPairwiseOpF(context, Intrinsic.X86Addps, Intrinsic.X86Addpd); + EmitSse2VectorPairwiseOpF(context, Intrinsic.X86Addps, Intrinsic.X86Addpd); } else { @@ -547,7 +554,7 @@ namespace ARMeilleure.Instructions { if (Optimizations.FastFP && Optimizations.UseSse2) { - EmitVectorPairwiseOpF(context, Intrinsic.X86Maxps, Intrinsic.X86Maxpd); + EmitSse2VectorPairwiseOpF(context, Intrinsic.X86Maxps, Intrinsic.X86Maxpd); } else { @@ -622,7 +629,7 @@ namespace ARMeilleure.Instructions { if (Optimizations.FastFP && Optimizations.UseSse2) { - EmitVectorPairwiseOpF(context, Intrinsic.X86Minps, Intrinsic.X86Minpd); + EmitSse2VectorPairwiseOpF(context, Intrinsic.X86Minps, Intrinsic.X86Minpd); } else { @@ -664,7 +671,7 @@ namespace ARMeilleure.Instructions res = context.VectorZeroUpper64(res); } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } else /* if (sizeF == 1) */ { @@ -672,7 +679,7 @@ namespace ARMeilleure.Instructions res = context.AddIntrinsic(Intrinsic.X86Addpd, d, res); - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } } else @@ -710,7 +717,7 @@ namespace ARMeilleure.Instructions res = context.VectorZeroUpper64(res); } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } else /* if (sizeF == 1) */ { @@ -721,7 +728,7 @@ namespace ARMeilleure.Instructions res = context.AddIntrinsic(Intrinsic.X86Mulpd, n, res); res = context.AddIntrinsic(Intrinsic.X86Addpd, d, res); - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } } else @@ -764,7 +771,7 @@ namespace ARMeilleure.Instructions res = context.VectorZeroUpper64(res); } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } else /* if (sizeF == 1) */ { @@ -772,7 +779,7 @@ namespace ARMeilleure.Instructions res = context.AddIntrinsic(Intrinsic.X86Subpd, d, res); - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } } else @@ -810,7 +817,7 @@ namespace ARMeilleure.Instructions res = context.VectorZeroUpper64(res); } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } else /* if (sizeF == 1) */ { @@ -821,7 +828,7 @@ namespace ARMeilleure.Instructions res = context.AddIntrinsic(Intrinsic.X86Mulpd, n, res); res = context.AddIntrinsic(Intrinsic.X86Subpd, d, res); - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } } else @@ -2028,9 +2035,16 @@ namespace ARMeilleure.Instructions public static void Smaxp_V(ArmEmitterContext context) { - Delegate dlg = new _S64_S64_S64(Math.Max); + if (Optimizations.UseSsse3) + { + EmitSsse3VectorPairwiseOp(context, X86PmaxsInstruction); + } + else + { + Delegate dlg = new _S64_S64_S64(Math.Max); - EmitVectorPairwiseOpSx(context, (op1, op2) => context.Call(dlg, op1, op2)); + EmitVectorPairwiseOpSx(context, (op1, op2) => context.Call(dlg, op1, op2)); + } } public static void Smaxv_V(ArmEmitterContext context) @@ -2070,9 +2084,16 @@ namespace ARMeilleure.Instructions public static void Sminp_V(ArmEmitterContext context) { - Delegate dlg = new _S64_S64_S64(Math.Min); + if (Optimizations.UseSsse3) + { + EmitSsse3VectorPairwiseOp(context, X86PminsInstruction); + } + else + { + Delegate dlg = new _S64_S64_S64(Math.Min); - EmitVectorPairwiseOpSx(context, (op1, op2) => context.Call(dlg, op1, op2)); + EmitVectorPairwiseOpSx(context, (op1, op2) => context.Call(dlg, op1, op2)); + } } public static void Sminv_V(ArmEmitterContext context) @@ -2653,9 +2674,16 @@ namespace ARMeilleure.Instructions public static void Umaxp_V(ArmEmitterContext context) { - Delegate dlg = new _U64_U64_U64(Math.Max); + if (Optimizations.UseSsse3) + { + EmitSsse3VectorPairwiseOp(context, X86PmaxuInstruction); + } + else + { + Delegate dlg = new _U64_U64_U64(Math.Max); - EmitVectorPairwiseOpZx(context, (op1, op2) => context.Call(dlg, op1, op2)); + EmitVectorPairwiseOpZx(context, (op1, op2) => context.Call(dlg, op1, op2)); + } } public static void Umaxv_V(ArmEmitterContext context) @@ -2695,9 +2723,16 @@ namespace ARMeilleure.Instructions public static void Uminp_V(ArmEmitterContext context) { - Delegate dlg = new _U64_U64_U64(Math.Min); + if (Optimizations.UseSsse3) + { + EmitSsse3VectorPairwiseOp(context, X86PminuInstruction); + } + else + { + Delegate dlg = new _U64_U64_U64(Math.Min); - EmitVectorPairwiseOpZx(context, (op1, op2) => context.Call(dlg, op1, op2)); + EmitVectorPairwiseOpZx(context, (op1, op2) => context.Call(dlg, op1, op2)); + } } public static void Uminv_V(ArmEmitterContext context) @@ -3020,7 +3055,9 @@ namespace ARMeilleure.Instructions int part = op.RegisterSize == RegisterSize.Simd128 ? elems : 0; - Operand res = part == 0 ? context.VectorZero() : context.Copy(GetVec(op.Rd)); + Operand d = GetVec(op.Rd); + + Operand res = part == 0 ? context.VectorZero() : context.Copy(d); long roundConst = 1L << (eSize - 1); @@ -3041,7 +3078,7 @@ namespace ARMeilleure.Instructions res = EmitVectorInsert(context, res, de, part + index, op.Size); } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } public static void EmitScalarRoundOpF(ArmEmitterContext context, FPRoundingMode roundMode) @@ -3124,12 +3161,12 @@ namespace ARMeilleure.Instructions Operand n = GetVec(op.Rn); Operand m = GetVec(op.Rm); - Operand nQNaNMask = EmitSse2VectorIsQNaNOpF(context, n); - Operand mQNaNMask = EmitSse2VectorIsQNaNOpF(context, m); - Operand nNum = context.Copy(n); Operand mNum = context.Copy(m); + Operand nQNaNMask = EmitSse2VectorIsQNaNOpF(context, nNum); + Operand mQNaNMask = EmitSse2VectorIsQNaNOpF(context, mNum); + int sizeF = op.Size & 1; if (sizeF == 0) diff --git a/ARMeilleure/Instructions/InstEmitSimdCvt.cs b/ARMeilleure/Instructions/InstEmitSimdCvt.cs index e2b6dbd7..30c1bd20 100644 --- a/ARMeilleure/Instructions/InstEmitSimdCvt.cs +++ b/ARMeilleure/Instructions/InstEmitSimdCvt.cs @@ -163,12 +163,26 @@ namespace ARMeilleure.Instructions public static void Fcvtms_Gp(ArmEmitterContext context) { - EmitFcvt_s_Gp(context, (op1) => EmitUnaryMathCall(context, MathF.Floor, Math.Floor, op1)); + if (Optimizations.UseSse41) + { + EmitSse41Fcvts_Gp(context, FPRoundingMode.TowardsMinusInfinity, isFixed: false); + } + else + { + EmitFcvt_s_Gp(context, (op1) => EmitUnaryMathCall(context, MathF.Floor, Math.Floor, op1)); + } } public static void Fcvtmu_Gp(ArmEmitterContext context) { - EmitFcvt_u_Gp(context, (op1) => EmitUnaryMathCall(context, MathF.Floor, Math.Floor, op1)); + if (Optimizations.UseSse41) + { + EmitSse41Fcvtu_Gp(context, FPRoundingMode.TowardsMinusInfinity, isFixed: false); + } + else + { + EmitFcvt_u_Gp(context, (op1) => EmitUnaryMathCall(context, MathF.Floor, Math.Floor, op1)); + } } public static void Fcvtn_V(ArmEmitterContext context) @@ -180,11 +194,10 @@ namespace ARMeilleure.Instructions if (Optimizations.UseSse2 && sizeF == 1) { Operand d = GetVec(op.Rd); - Operand n = GetVec(op.Rn); - Operand res = context.AddIntrinsic(Intrinsic.X86Movlhps, d, context.VectorZero()); + Operand res = context.VectorZeroUpper64(d); - Operand nInt = context.AddIntrinsic(Intrinsic.X86Cvtpd2ps, n); + Operand nInt = context.AddIntrinsic(Intrinsic.X86Cvtpd2ps, GetVec(op.Rn)); nInt = context.AddIntrinsic(Intrinsic.X86Movlhps, nInt, nInt); @@ -194,7 +207,7 @@ namespace ARMeilleure.Instructions res = context.AddIntrinsic(movInst, res, nInt); - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } else { @@ -204,7 +217,9 @@ namespace ARMeilleure.Instructions int part = op.RegisterSize == RegisterSize.Simd128 ? elems : 0; - Operand res = part == 0 ? context.VectorZero() : context.Copy(GetVec(op.Rd)); + Operand d = GetVec(op.Rd); + + Operand res = part == 0 ? context.VectorZero() : context.Copy(d); for (int index = 0; index < elems; index++) { @@ -228,7 +243,7 @@ namespace ARMeilleure.Instructions } } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } } @@ -282,22 +297,50 @@ namespace ARMeilleure.Instructions public static void Fcvtps_Gp(ArmEmitterContext context) { - EmitFcvt_s_Gp(context, (op1) => EmitUnaryMathCall(context, MathF.Ceiling, Math.Ceiling, op1)); + if (Optimizations.UseSse41) + { + EmitSse41Fcvts_Gp(context, FPRoundingMode.TowardsPlusInfinity, isFixed: false); + } + else + { + EmitFcvt_s_Gp(context, (op1) => EmitUnaryMathCall(context, MathF.Ceiling, Math.Ceiling, op1)); + } } public static void Fcvtpu_Gp(ArmEmitterContext context) { - EmitFcvt_u_Gp(context, (op1) => EmitUnaryMathCall(context, MathF.Ceiling, Math.Ceiling, op1)); + if (Optimizations.UseSse41) + { + EmitSse41Fcvtu_Gp(context, FPRoundingMode.TowardsPlusInfinity, isFixed: false); + } + else + { + EmitFcvt_u_Gp(context, (op1) => EmitUnaryMathCall(context, MathF.Ceiling, Math.Ceiling, op1)); + } } public static void Fcvtzs_Gp(ArmEmitterContext context) { - EmitFcvt_s_Gp(context, (op1) => op1); + if (Optimizations.UseSse41) + { + EmitSse41Fcvts_Gp(context, FPRoundingMode.TowardsZero, isFixed: false); + } + else + { + EmitFcvt_s_Gp(context, (op1) => op1); + } } public static void Fcvtzs_Gp_Fixed(ArmEmitterContext context) { - EmitFcvtzs_Gp_Fixed(context); + if (Optimizations.UseSse41) + { + EmitSse41Fcvts_Gp(context, FPRoundingMode.TowardsZero, isFixed: true); + } + else + { + EmitFcvtzs_Gp_Fixed(context); + } } public static void Fcvtzs_S(ArmEmitterContext context) @@ -338,12 +381,26 @@ namespace ARMeilleure.Instructions public static void Fcvtzu_Gp(ArmEmitterContext context) { - EmitFcvt_u_Gp(context, (op1) => op1); + if (Optimizations.UseSse41) + { + EmitSse41Fcvtu_Gp(context, FPRoundingMode.TowardsZero, isFixed: false); + } + else + { + EmitFcvt_u_Gp(context, (op1) => op1); + } } public static void Fcvtzu_Gp_Fixed(ArmEmitterContext context) { - EmitFcvtzu_Gp_Fixed(context); + if (Optimizations.UseSse41) + { + EmitSse41Fcvtu_Gp(context, FPRoundingMode.TowardsZero, isFixed: true); + } + else + { + EmitFcvtzu_Gp_Fixed(context); + } } public static void Fcvtzu_S(ArmEmitterContext context) @@ -418,16 +475,16 @@ namespace ARMeilleure.Instructions public static void Scvtf_S(ArmEmitterContext context) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; - - int sizeF = op.Size & 1; - - if (Optimizations.UseSse2 && sizeF == 0) + if (Optimizations.UseSse2) { EmitSse2Scvtf(context, scalar: true); } else { + OpCodeSimd op = (OpCodeSimd)context.CurrOp; + + int sizeF = op.Size & 1; + Operand res = EmitVectorLongExtract(context, op.Rn, 0, sizeF + 2); res = EmitFPConvert(context, res, op.Size, signed: true); @@ -438,11 +495,7 @@ namespace ARMeilleure.Instructions public static void Scvtf_V(ArmEmitterContext context) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; - - int sizeF = op.Size & 1; - - if (Optimizations.UseSse2 && sizeF == 0) + if (Optimizations.UseSse2) { EmitSse2Scvtf(context, scalar: false); } @@ -454,12 +507,7 @@ namespace ARMeilleure.Instructions public static void Scvtf_V_Fixed(ArmEmitterContext context) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; - - // sizeF == ((OpCodeSimdShImm64)op).Size - 2 - int sizeF = op.Size & 1; - - if (Optimizations.UseSse2 && sizeF == 0) + if (Optimizations.UseSse2) { EmitSse2Scvtf(context, scalar: false); } @@ -495,16 +543,16 @@ namespace ARMeilleure.Instructions public static void Ucvtf_S(ArmEmitterContext context) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; - - int sizeF = op.Size & 1; - - if (Optimizations.UseSse2 && sizeF == 0) + if (Optimizations.UseSse2) { EmitSse2Ucvtf(context, scalar: true); } else { + OpCodeSimd op = (OpCodeSimd)context.CurrOp; + + int sizeF = op.Size & 1; + Operand ne = EmitVectorLongExtract(context, op.Rn, 0, sizeF + 2); Operand res = EmitFPConvert(context, ne, sizeF, signed: false); @@ -515,11 +563,7 @@ namespace ARMeilleure.Instructions public static void Ucvtf_V(ArmEmitterContext context) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; - - int sizeF = op.Size & 1; - - if (Optimizations.UseSse2 && sizeF == 0) + if (Optimizations.UseSse2) { EmitSse2Ucvtf(context, scalar: false); } @@ -531,12 +575,7 @@ namespace ARMeilleure.Instructions public static void Ucvtf_V_Fixed(ArmEmitterContext context) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; - - // sizeF == ((OpCodeSimdShImm)op).Size - 2 - int sizeF = op.Size & 1; - - if (Optimizations.UseSse2 && sizeF == 0) + if (Optimizations.UseSse2) { EmitSse2Ucvtf(context, scalar: false); } @@ -830,43 +869,69 @@ namespace ARMeilleure.Instructions } } - private static void EmitSse41Fcvts(ArmEmitterContext context, FPRoundingMode roundMode, bool scalar) + private static Operand EmitSse2CvtDoubleToInt64OpF(ArmEmitterContext context, Operand opF, bool scalar) + { + Debug.Assert(opF.Type == OperandType.V128); + + Operand longL = context.AddIntrinsicLong (Intrinsic.X86Cvtsd2si, opF); // opFL + Operand res = context.VectorCreateScalar(longL); + + if (!scalar) + { + Operand opFH = context.AddIntrinsic (Intrinsic.X86Movhlps, res, opF); // res doesn't matter. + Operand longH = context.AddIntrinsicLong (Intrinsic.X86Cvtsd2si, opFH); + Operand resH = context.VectorCreateScalar(longH); + res = context.AddIntrinsic (Intrinsic.X86Movlhps, res, resH); + } + + return res; + } + + private static Operand EmitSse2CvtInt64ToDoubleOp(ArmEmitterContext context, Operand op, bool scalar) + { + Debug.Assert(op.Type == OperandType.V128); + + Operand longL = context.AddIntrinsicLong(Intrinsic.X86Cvtsi2si, op); // opL + Operand res = context.AddIntrinsic (Intrinsic.X86Cvtsi2sd, context.VectorZero(), longL); + + if (!scalar) + { + Operand opH = context.AddIntrinsic (Intrinsic.X86Movhlps, res, op); // res doesn't matter. + Operand longH = context.AddIntrinsicLong(Intrinsic.X86Cvtsi2si, opH); + Operand resH = context.AddIntrinsic (Intrinsic.X86Cvtsi2sd, res, longH); // res doesn't matter. + res = context.AddIntrinsic (Intrinsic.X86Movlhps, res, resH); + } + + return res; + } + + private static void EmitSse2Scvtf(ArmEmitterContext context, bool scalar) { OpCodeSimd op = (OpCodeSimd)context.CurrOp; Operand n = GetVec(op.Rn); - // sizeF == ((OpCodeSimdShImm64)op).Size - 2 + // sizeF == ((OpCodeSimdShImm)op).Size - 2 int sizeF = op.Size & 1; if (sizeF == 0) { - Operand nMask = context.AddIntrinsic(Intrinsic.X86Cmpps, n, n, Const((int)CmpCondition.OrderedQ)); - - Operand nScaled = context.AddIntrinsic(Intrinsic.X86Pand, nMask, n); + Operand res = context.AddIntrinsic(Intrinsic.X86Cvtdq2ps, n); if (op is OpCodeSimdShImm fixedOp) { int fBits = GetImmShr(fixedOp); - // BitConverter.Int32BitsToSingle(fpScaled) == MathF.Pow(2f, fBits) - int fpScaled = 0x3F800000 + fBits * 0x800000; + // BitConverter.Int32BitsToSingle(fpScaled) == 1f / MathF.Pow(2f, fBits) + int fpScaled = 0x3F800000 - fBits * 0x800000; - Operand scale = X86GetAllElements(context, fpScaled); + Operand fpScaledMask = scalar + ? X86GetScalar (context, fpScaled) + : X86GetAllElements(context, fpScaled); - nScaled = context.AddIntrinsic(Intrinsic.X86Mulps, nScaled, scale); + res = context.AddIntrinsic(Intrinsic.X86Mulps, res, fpScaledMask); } - Operand nRnd = context.AddIntrinsic(Intrinsic.X86Roundps, nScaled, Const(X86GetRoundControl(roundMode))); - - Operand nInt = context.AddIntrinsic(Intrinsic.X86Cvtps2dq, nRnd); - - Operand mask = X86GetAllElements(context, 0x4F000000); // 2.14748365E9f (2147483648) - - Operand mask2 = context.AddIntrinsic(Intrinsic.X86Cmpps, nRnd, mask, Const((int)CmpCondition.NotLessThan)); - - Operand res = context.AddIntrinsic(Intrinsic.X86Pxor, nInt, mask2); - if (scalar) { res = context.VectorZeroUpper96(res); @@ -880,9 +945,175 @@ namespace ARMeilleure.Instructions } else /* if (sizeF == 1) */ { - Operand nMask = context.AddIntrinsic(Intrinsic.X86Cmppd, n, n, Const((int)CmpCondition.OrderedQ)); + Operand res = EmitSse2CvtInt64ToDoubleOp(context, n, scalar); - Operand nScaled = context.AddIntrinsic(Intrinsic.X86Pand, nMask, n); + if (op is OpCodeSimdShImm fixedOp) + { + int fBits = GetImmShr(fixedOp); + + // BitConverter.Int64BitsToDouble(fpScaled) == 1d / Math.Pow(2d, fBits) + long fpScaled = 0x3FF0000000000000L - fBits * 0x10000000000000L; + + Operand fpScaledMask = scalar + ? X86GetScalar (context, fpScaled) + : X86GetAllElements(context, fpScaled); + + res = context.AddIntrinsic(Intrinsic.X86Mulpd, res, fpScaledMask); + } + + if (scalar) + { + res = context.VectorZeroUpper64(res); + } + + context.Copy(GetVec(op.Rd), res); + } + } + + private static void EmitSse2Ucvtf(ArmEmitterContext context, bool scalar) + { + OpCodeSimd op = (OpCodeSimd)context.CurrOp; + + Operand n = GetVec(op.Rn); + + // sizeF == ((OpCodeSimdShImm)op).Size - 2 + int sizeF = op.Size & 1; + + if (sizeF == 0) + { + Operand mask = scalar // 65536.000f (1 << 16) + ? X86GetScalar (context, 0x47800000) + : X86GetAllElements(context, 0x47800000); + + Operand res = context.AddIntrinsic(Intrinsic.X86Psrld, n, Const(16)); + res = context.AddIntrinsic(Intrinsic.X86Cvtdq2ps, res); + res = context.AddIntrinsic(Intrinsic.X86Mulps, res, mask); + + Operand res2 = context.AddIntrinsic(Intrinsic.X86Pslld, n, Const(16)); + res2 = context.AddIntrinsic(Intrinsic.X86Psrld, res2, Const(16)); + res2 = context.AddIntrinsic(Intrinsic.X86Cvtdq2ps, res2); + + res = context.AddIntrinsic(Intrinsic.X86Addps, res, res2); + + if (op is OpCodeSimdShImm fixedOp) + { + int fBits = GetImmShr(fixedOp); + + // BitConverter.Int32BitsToSingle(fpScaled) == 1f / MathF.Pow(2f, fBits) + int fpScaled = 0x3F800000 - fBits * 0x800000; + + Operand fpScaledMask = scalar + ? X86GetScalar (context, fpScaled) + : X86GetAllElements(context, fpScaled); + + res = context.AddIntrinsic(Intrinsic.X86Mulps, res, fpScaledMask); + } + + if (scalar) + { + res = context.VectorZeroUpper96(res); + } + else if (op.RegisterSize == RegisterSize.Simd64) + { + res = context.VectorZeroUpper64(res); + } + + context.Copy(GetVec(op.Rd), res); + } + else /* if (sizeF == 1) */ + { + Operand mask = scalar // 4294967296.0000000d (1L << 32) + ? X86GetScalar (context, 0x41F0000000000000L) + : X86GetAllElements(context, 0x41F0000000000000L); + + Operand res = context.AddIntrinsic (Intrinsic.X86Psrlq, n, Const(32)); + res = EmitSse2CvtInt64ToDoubleOp(context, res, scalar); + res = context.AddIntrinsic (Intrinsic.X86Mulpd, res, mask); + + Operand res2 = context.AddIntrinsic (Intrinsic.X86Psllq, n, Const(32)); + res2 = context.AddIntrinsic (Intrinsic.X86Psrlq, res2, Const(32)); + res2 = EmitSse2CvtInt64ToDoubleOp(context, res2, scalar); + + res = context.AddIntrinsic(Intrinsic.X86Addpd, res, res2); + + if (op is OpCodeSimdShImm fixedOp) + { + int fBits = GetImmShr(fixedOp); + + // BitConverter.Int64BitsToDouble(fpScaled) == 1d / Math.Pow(2d, fBits) + long fpScaled = 0x3FF0000000000000L - fBits * 0x10000000000000L; + + Operand fpScaledMask = scalar + ? X86GetScalar (context, fpScaled) + : X86GetAllElements(context, fpScaled); + + res = context.AddIntrinsic(Intrinsic.X86Mulpd, res, fpScaledMask); + } + + if (scalar) + { + res = context.VectorZeroUpper64(res); + } + + context.Copy(GetVec(op.Rd), res); + } + } + + private static void EmitSse41Fcvts(ArmEmitterContext context, FPRoundingMode roundMode, bool scalar) + { + OpCodeSimd op = (OpCodeSimd)context.CurrOp; + + Operand n = GetVec(op.Rn); + + // sizeF == ((OpCodeSimdShImm)op).Size - 2 + int sizeF = op.Size & 1; + + if (sizeF == 0) + { + Operand nRes = context.AddIntrinsic(Intrinsic.X86Cmpps, n, n, Const((int)CmpCondition.OrderedQ)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, n); + + if (op is OpCodeSimdShImm fixedOp) + { + int fBits = GetImmShr(fixedOp); + + // BitConverter.Int32BitsToSingle(fpScaled) == MathF.Pow(2f, fBits) + int fpScaled = 0x3F800000 + fBits * 0x800000; + + Operand fpScaledMask = scalar + ? X86GetScalar (context, fpScaled) + : X86GetAllElements(context, fpScaled); + + nRes = context.AddIntrinsic(Intrinsic.X86Mulps, nRes, fpScaledMask); + } + + nRes = context.AddIntrinsic(Intrinsic.X86Roundps, nRes, Const(X86GetRoundControl(roundMode))); + + Operand nInt = context.AddIntrinsic(Intrinsic.X86Cvtps2dq, nRes); + + Operand fpMaxValMask = scalar // 2.14748365E9f (2147483648) + ? X86GetScalar (context, 0x4F000000) + : X86GetAllElements(context, 0x4F000000); + + nRes = context.AddIntrinsic(Intrinsic.X86Cmpps, nRes, fpMaxValMask, Const((int)CmpCondition.NotLessThan)); + + Operand dRes = context.AddIntrinsic(Intrinsic.X86Pxor, nInt, nRes); + + if (scalar) + { + dRes = context.VectorZeroUpper96(dRes); + } + else if (op.RegisterSize == RegisterSize.Simd64) + { + dRes = context.VectorZeroUpper64(dRes); + } + + context.Copy(GetVec(op.Rd), dRes); + } + else /* if (sizeF == 1) */ + { + Operand nRes = context.AddIntrinsic(Intrinsic.X86Cmppd, n, n, Const((int)CmpCondition.OrderedQ)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, n); if (op is OpCodeSimdShImm fixedOp) { @@ -891,41 +1122,31 @@ namespace ARMeilleure.Instructions // BitConverter.Int64BitsToDouble(fpScaled) == Math.Pow(2d, fBits) long fpScaled = 0x3FF0000000000000L + fBits * 0x10000000000000L; - Operand scale = X86GetAllElements(context, fpScaled); + Operand fpScaledMask = scalar + ? X86GetScalar (context, fpScaled) + : X86GetAllElements(context, fpScaled); - nScaled = context.AddIntrinsic(Intrinsic.X86Mulpd, nScaled, scale); + nRes = context.AddIntrinsic(Intrinsic.X86Mulpd, nRes, fpScaledMask); } - Operand nRnd = context.AddIntrinsic(Intrinsic.X86Roundpd, nScaled, Const(X86GetRoundControl(roundMode))); + nRes = context.AddIntrinsic(Intrinsic.X86Roundpd, nRes, Const(X86GetRoundControl(roundMode))); - Operand high; + Operand nLong = EmitSse2CvtDoubleToInt64OpF(context, nRes, scalar); - if (!scalar) - { - high = context.AddIntrinsic(Intrinsic.X86Unpckhpd, nRnd, nRnd); - high = context.AddIntrinsicLong(Intrinsic.X86Cvtsd2si, high); - } - else - { - high = Const(0L); - } + Operand fpMaxValMask = scalar // 9.2233720368547760E18d (9223372036854775808) + ? X86GetScalar (context, 0x43E0000000000000L) + : X86GetAllElements(context, 0x43E0000000000000L); - Operand low = context.AddIntrinsicLong(Intrinsic.X86Cvtsd2si, nRnd); + nRes = context.AddIntrinsic(Intrinsic.X86Cmppd, nRes, fpMaxValMask, Const((int)CmpCondition.NotLessThan)); - Operand nInt = EmitVectorLongCreate(context, low, high); - - Operand mask = X86GetAllElements(context, 0x43E0000000000000L); // 9.2233720368547760E18d (9223372036854775808) - - Operand mask2 = context.AddIntrinsic(Intrinsic.X86Cmppd, nRnd, mask, Const((int)CmpCondition.NotLessThan)); - - Operand res = context.AddIntrinsic(Intrinsic.X86Pxor, nInt, mask2); + Operand dRes = context.AddIntrinsic(Intrinsic.X86Pxor, nLong, nRes); if (scalar) { - res = context.VectorZeroUpper64(res); + dRes = context.VectorZeroUpper64(dRes); } - context.Copy(GetVec(op.Rd), res); + context.Copy(GetVec(op.Rd), dRes); } } @@ -940,9 +1161,8 @@ namespace ARMeilleure.Instructions if (sizeF == 0) { - Operand nMask = context.AddIntrinsic(Intrinsic.X86Cmpps, n, n, Const((int)CmpCondition.OrderedQ)); - - Operand nScaled = context.AddIntrinsic(Intrinsic.X86Pand, nMask, n); + Operand nRes = context.AddIntrinsic(Intrinsic.X86Cmpps, n, n, Const((int)CmpCondition.OrderedQ)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, n); if (op is OpCodeSimdShImm fixedOp) { @@ -951,50 +1171,53 @@ namespace ARMeilleure.Instructions // BitConverter.Int32BitsToSingle(fpScaled) == MathF.Pow(2f, fBits) int fpScaled = 0x3F800000 + fBits * 0x800000; - Operand scale = X86GetAllElements(context, fpScaled); + Operand fpScaledMask = scalar + ? X86GetScalar (context, fpScaled) + : X86GetAllElements(context, fpScaled); - nScaled = context.AddIntrinsic(Intrinsic.X86Mulps, nScaled, scale); + nRes = context.AddIntrinsic(Intrinsic.X86Mulps, nRes, fpScaledMask); } - Operand nRnd = context.AddIntrinsic(Intrinsic.X86Roundps, nScaled, Const(X86GetRoundControl(roundMode))); + nRes = context.AddIntrinsic(Intrinsic.X86Roundps, nRes, Const(X86GetRoundControl(roundMode))); - Operand nRndMask = context.AddIntrinsic(Intrinsic.X86Cmpps, nRnd, context.VectorZero(), Const((int)CmpCondition.NotLessThanOrEqual)); + Operand zero = context.VectorZero(); - Operand nRndMasked = context.AddIntrinsic(Intrinsic.X86Pand, nRnd, nRndMask); + Operand nCmp = context.AddIntrinsic(Intrinsic.X86Cmpps, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp); - Operand nInt = context.AddIntrinsic(Intrinsic.X86Cvtps2dq, nRndMasked); + Operand fpMaxValMask = scalar // 2.14748365E9f (2147483648) + ? X86GetScalar (context, 0x4F000000) + : X86GetAllElements(context, 0x4F000000); - Operand mask = X86GetAllElements(context, 0x4F000000); // 2.14748365E9f (2147483648) + Operand nInt = context.AddIntrinsic(Intrinsic.X86Cvtps2dq, nRes); - Operand res = context.AddIntrinsic(Intrinsic.X86Subps, nRndMasked, mask); + nRes = context.AddIntrinsic(Intrinsic.X86Subps, nRes, fpMaxValMask); - Operand mask2 = context.AddIntrinsic(Intrinsic.X86Cmpps, res, context.VectorZero(), Const((int)CmpCondition.NotLessThanOrEqual)); + nCmp = context.AddIntrinsic(Intrinsic.X86Cmpps, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp); - Operand resMasked = context.AddIntrinsic(Intrinsic.X86Pand, res, mask2); + Operand nInt2 = context.AddIntrinsic(Intrinsic.X86Cvtps2dq, nRes); - res = context.AddIntrinsic(Intrinsic.X86Cvtps2dq, resMasked); + nRes = context.AddIntrinsic(Intrinsic.X86Cmpps, nRes, fpMaxValMask, Const((int)CmpCondition.NotLessThan)); - Operand mask3 = context.AddIntrinsic(Intrinsic.X86Cmpps, resMasked, mask, Const((int)CmpCondition.NotLessThan)); - - res = context.AddIntrinsic(Intrinsic.X86Pxor, res, mask3); - res = context.AddIntrinsic(Intrinsic.X86Paddd, res, nInt); + Operand dRes = context.AddIntrinsic(Intrinsic.X86Pxor, nInt2, nRes); + dRes = context.AddIntrinsic(Intrinsic.X86Paddd, dRes, nInt); if (scalar) { - res = context.VectorZeroUpper96(res); + dRes = context.VectorZeroUpper96(dRes); } else if (op.RegisterSize == RegisterSize.Simd64) { - res = context.VectorZeroUpper64(res); + dRes = context.VectorZeroUpper64(dRes); } - context.Copy(GetVec(op.Rd), res); + context.Copy(GetVec(op.Rd), dRes); } else /* if (sizeF == 1) */ { - Operand nMask = context.AddIntrinsic(Intrinsic.X86Cmppd, n, n, Const((int)CmpCondition.OrderedQ)); - - Operand nScaled = context.AddIntrinsic(Intrinsic.X86Pand, nMask, n); + Operand nRes = context.AddIntrinsic(Intrinsic.X86Cmppd, n, n, Const((int)CmpCondition.OrderedQ)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, n); if (op is OpCodeSimdShImm fixedOp) { @@ -1003,140 +1226,251 @@ namespace ARMeilleure.Instructions // BitConverter.Int64BitsToDouble(fpScaled) == Math.Pow(2d, fBits) long fpScaled = 0x3FF0000000000000L + fBits * 0x10000000000000L; - Operand scale = X86GetAllElements(context, fpScaled); + Operand fpScaledMask = scalar + ? X86GetScalar (context, fpScaled) + : X86GetAllElements(context, fpScaled); - nScaled = context.AddIntrinsic(Intrinsic.X86Mulpd, nScaled, scale); + nRes = context.AddIntrinsic(Intrinsic.X86Mulpd, nRes, fpScaledMask); } - Operand nRnd = context.AddIntrinsic(Intrinsic.X86Roundpd, nScaled, Const(X86GetRoundControl(roundMode))); + nRes = context.AddIntrinsic(Intrinsic.X86Roundpd, nRes, Const(X86GetRoundControl(roundMode))); - Operand nRndMask = context.AddIntrinsic(Intrinsic.X86Cmppd, nRnd, context.VectorZero(), Const((int)CmpCondition.NotLessThanOrEqual)); + Operand zero = context.VectorZero(); - Operand nRndMasked = context.AddIntrinsic(Intrinsic.X86Pand, nRnd, nRndMask); + Operand nCmp = context.AddIntrinsic(Intrinsic.X86Cmppd, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp); - Operand high; + Operand fpMaxValMask = scalar // 9.2233720368547760E18d (9223372036854775808) + ? X86GetScalar (context, 0x43E0000000000000L) + : X86GetAllElements(context, 0x43E0000000000000L); - if (!scalar) - { - high = context.AddIntrinsic(Intrinsic.X86Unpckhpd, nRndMasked, nRndMasked); - high = context.AddIntrinsicLong(Intrinsic.X86Cvtsd2si, high); - } - else - { - high = Const(0L); - } + Operand nLong = EmitSse2CvtDoubleToInt64OpF(context, nRes, scalar); - Operand low = context.AddIntrinsicLong(Intrinsic.X86Cvtsd2si, nRndMasked); + nRes = context.AddIntrinsic(Intrinsic.X86Subpd, nRes, fpMaxValMask); - Operand nInt = EmitVectorLongCreate(context, low, high); + nCmp = context.AddIntrinsic(Intrinsic.X86Cmppd, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp); - Operand mask = X86GetAllElements(context, 0x43E0000000000000L); // 9.2233720368547760E18d (9223372036854775808) + Operand nLong2 = EmitSse2CvtDoubleToInt64OpF(context, nRes, scalar); - Operand res = context.AddIntrinsic(Intrinsic.X86Subpd, nRndMasked, mask); + nRes = context.AddIntrinsic(Intrinsic.X86Cmppd, nRes, fpMaxValMask, Const((int)CmpCondition.NotLessThan)); - Operand mask2 = context.AddIntrinsic(Intrinsic.X86Cmppd, res, context.VectorZero(), Const((int)CmpCondition.NotLessThanOrEqual)); - - Operand resMasked = context.AddIntrinsic(Intrinsic.X86Pand, res, mask2); - - if (!scalar) - { - high = context.AddIntrinsic(Intrinsic.X86Unpckhpd, resMasked, resMasked); - high = context.AddIntrinsicLong(Intrinsic.X86Cvtsd2si, high); - } - - low = context.AddIntrinsicLong(Intrinsic.X86Cvtsd2si, resMasked); - - res = EmitVectorLongCreate(context, low, high); - - Operand mask3 = context.AddIntrinsic(Intrinsic.X86Cmppd, resMasked, mask, Const((int)CmpCondition.NotLessThan)); - - res = context.AddIntrinsic(Intrinsic.X86Pxor, res, mask3); - res = context.AddIntrinsic(Intrinsic.X86Paddq, res, nInt); + Operand dRes = context.AddIntrinsic(Intrinsic.X86Pxor, nLong2, nRes); + dRes = context.AddIntrinsic(Intrinsic.X86Paddq, dRes, nLong); if (scalar) { - res = context.VectorZeroUpper64(res); + dRes = context.VectorZeroUpper64(dRes); } - context.Copy(GetVec(op.Rd), res); + context.Copy(GetVec(op.Rd), dRes); } } - private static void EmitSse2Scvtf(ArmEmitterContext context, bool scalar) + private static void EmitSse41Fcvts_Gp(ArmEmitterContext context, FPRoundingMode roundMode, bool isFixed) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; + OpCodeSimdCvt op = (OpCodeSimdCvt)context.CurrOp; Operand n = GetVec(op.Rn); - Operand res = context.AddIntrinsic(Intrinsic.X86Cvtdq2ps, n); - - if (op is OpCodeSimdShImm fixedOp) + if (op.Size == 0) { - int fBits = GetImmShr(fixedOp); + Operand nRes = context.AddIntrinsic(Intrinsic.X86Cmpss, n, n, Const((int)CmpCondition.OrderedQ)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, n); - // BitConverter.Int32BitsToSingle(fpScaled) == 1f / MathF.Pow(2f, fBits) - int fpScaled = 0x3F800000 - fBits * 0x800000; + if (isFixed) + { + // BitConverter.Int32BitsToSingle(fpScaled) == MathF.Pow(2f, op.FBits) + int fpScaled = 0x3F800000 + op.FBits * 0x800000; - Operand scale = X86GetAllElements(context, fpScaled); + Operand fpScaledMask = X86GetScalar(context, fpScaled); - res = context.AddIntrinsic(Intrinsic.X86Mulps, res, scale); + nRes = context.AddIntrinsic(Intrinsic.X86Mulss, nRes, fpScaledMask); + } + + nRes = context.AddIntrinsic(Intrinsic.X86Roundss, nRes, Const(X86GetRoundControl(roundMode))); + + Operand nIntOrLong = op.RegisterSize == RegisterSize.Int32 + ? context.AddIntrinsicInt (Intrinsic.X86Cvtss2si, nRes) + : context.AddIntrinsicLong(Intrinsic.X86Cvtss2si, nRes); + + int fpMaxVal = op.RegisterSize == RegisterSize.Int32 + ? 0x4F000000 // 2.14748365E9f (2147483648) + : 0x5F000000; // 9.223372E18f (9223372036854775808) + + Operand fpMaxValMask = X86GetScalar(context, fpMaxVal); + + nRes = context.AddIntrinsic(Intrinsic.X86Cmpss, nRes, fpMaxValMask, Const((int)CmpCondition.NotLessThan)); + + Operand nInt = context.AddIntrinsicInt(Intrinsic.X86Cvtsi2si, nRes); + + if (op.RegisterSize == RegisterSize.Int64) + { + nInt = context.SignExtend32(OperandType.I64, nInt); + } + + Operand dRes = context.BitwiseExclusiveOr(nIntOrLong, nInt); + + SetIntOrZR(context, op.Rd, dRes); } - - if (scalar) + else /* if (op.Size == 1) */ { - res = context.VectorZeroUpper96(res); - } - else if (op.RegisterSize == RegisterSize.Simd64) - { - res = context.VectorZeroUpper64(res); - } + Operand nRes = context.AddIntrinsic(Intrinsic.X86Cmpsd, n, n, Const((int)CmpCondition.OrderedQ)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, n); - context.Copy(GetVec(op.Rd), res); + if (isFixed) + { + // BitConverter.Int64BitsToDouble(fpScaled) == Math.Pow(2d, op.FBits) + long fpScaled = 0x3FF0000000000000L + op.FBits * 0x10000000000000L; + + Operand fpScaledMask = X86GetScalar(context, fpScaled); + + nRes = context.AddIntrinsic(Intrinsic.X86Mulsd, nRes, fpScaledMask); + } + + nRes = context.AddIntrinsic(Intrinsic.X86Roundsd, nRes, Const(X86GetRoundControl(roundMode))); + + Operand nIntOrLong = op.RegisterSize == RegisterSize.Int32 + ? context.AddIntrinsicInt (Intrinsic.X86Cvtsd2si, nRes) + : context.AddIntrinsicLong(Intrinsic.X86Cvtsd2si, nRes); + + long fpMaxVal = op.RegisterSize == RegisterSize.Int32 + ? 0x41E0000000000000L // 2147483648.0000000d (2147483648) + : 0x43E0000000000000L; // 9.2233720368547760E18d (9223372036854775808) + + Operand fpMaxValMask = X86GetScalar(context, fpMaxVal); + + nRes = context.AddIntrinsic(Intrinsic.X86Cmpsd, nRes, fpMaxValMask, Const((int)CmpCondition.NotLessThan)); + + Operand nLong = context.AddIntrinsicLong(Intrinsic.X86Cvtsi2si, nRes); + + if (op.RegisterSize == RegisterSize.Int32) + { + nLong = context.ConvertI64ToI32(nLong); + } + + Operand dRes = context.BitwiseExclusiveOr(nIntOrLong, nLong); + + SetIntOrZR(context, op.Rd, dRes); + } } - private static void EmitSse2Ucvtf(ArmEmitterContext context, bool scalar) + private static void EmitSse41Fcvtu_Gp(ArmEmitterContext context, FPRoundingMode roundMode, bool isFixed) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; + OpCodeSimdCvt op = (OpCodeSimdCvt)context.CurrOp; Operand n = GetVec(op.Rn); - Operand res = context.AddIntrinsic(Intrinsic.X86Psrld, n, Const(16)); - - res = context.AddIntrinsic(Intrinsic.X86Cvtdq2ps, res); - - Operand mask = X86GetAllElements(context, 0x47800000); // 65536.0f (1 << 16) - - res = context.AddIntrinsic(Intrinsic.X86Mulps, res, mask); - - Operand res2 = context.AddIntrinsic(Intrinsic.X86Pslld, n, Const(16)); - - res2 = context.AddIntrinsic(Intrinsic.X86Psrld, res2, Const(16)); - res2 = context.AddIntrinsic(Intrinsic.X86Cvtdq2ps, res2); - - res = context.AddIntrinsic(Intrinsic.X86Addps, res, res2); - - if (op is OpCodeSimdShImm fixedOp) + if (op.Size == 0) { - int fBits = GetImmShr(fixedOp); + Operand nRes = context.AddIntrinsic(Intrinsic.X86Cmpss, n, n, Const((int)CmpCondition.OrderedQ)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, n); - // BitConverter.Int32BitsToSingle(fpScaled) == 1f / MathF.Pow(2f, fBits) - int fpScaled = 0x3F800000 - fBits * 0x800000; + if (isFixed) + { + // BitConverter.Int32BitsToSingle(fpScaled) == MathF.Pow(2f, op.FBits) + int fpScaled = 0x3F800000 + op.FBits * 0x800000; - Operand scale = X86GetAllElements(context, fpScaled); + Operand fpScaledMask = X86GetScalar(context, fpScaled); - res = context.AddIntrinsic(Intrinsic.X86Mulps, res, scale); + nRes = context.AddIntrinsic(Intrinsic.X86Mulss, nRes, fpScaledMask); + } + + nRes = context.AddIntrinsic(Intrinsic.X86Roundss, nRes, Const(X86GetRoundControl(roundMode))); + + Operand zero = context.VectorZero(); + + Operand nCmp = context.AddIntrinsic(Intrinsic.X86Cmpss, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp); + + int fpMaxVal = op.RegisterSize == RegisterSize.Int32 + ? 0x4F000000 // 2.14748365E9f (2147483648) + : 0x5F000000; // 9.223372E18f (9223372036854775808) + + Operand fpMaxValMask = X86GetScalar(context, fpMaxVal); + + Operand nIntOrLong = op.RegisterSize == RegisterSize.Int32 + ? context.AddIntrinsicInt (Intrinsic.X86Cvtss2si, nRes) + : context.AddIntrinsicLong(Intrinsic.X86Cvtss2si, nRes); + + nRes = context.AddIntrinsic(Intrinsic.X86Subss, nRes, fpMaxValMask); + + nCmp = context.AddIntrinsic(Intrinsic.X86Cmpss, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp); + + Operand nIntOrLong2 = op.RegisterSize == RegisterSize.Int32 + ? context.AddIntrinsicInt (Intrinsic.X86Cvtss2si, nRes) + : context.AddIntrinsicLong(Intrinsic.X86Cvtss2si, nRes); + + nRes = context.AddIntrinsic(Intrinsic.X86Cmpss, nRes, fpMaxValMask, Const((int)CmpCondition.NotLessThan)); + + Operand nInt = context.AddIntrinsicInt(Intrinsic.X86Cvtsi2si, nRes); + + if (op.RegisterSize == RegisterSize.Int64) + { + nInt = context.SignExtend32(OperandType.I64, nInt); + } + + Operand dRes = context.BitwiseExclusiveOr(nIntOrLong2, nInt); + dRes = context.Add(dRes, nIntOrLong); + + SetIntOrZR(context, op.Rd, dRes); } - - if (scalar) + else /* if (op.Size == 1) */ { - res = context.VectorZeroUpper96(res); - } - else if (op.RegisterSize == RegisterSize.Simd64) - { - res = context.VectorZeroUpper64(res); - } + Operand nRes = context.AddIntrinsic(Intrinsic.X86Cmpsd, n, n, Const((int)CmpCondition.OrderedQ)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, n); - context.Copy(GetVec(op.Rd), res); + if (isFixed) + { + // BitConverter.Int64BitsToDouble(fpScaled) == Math.Pow(2d, op.FBits) + long fpScaled = 0x3FF0000000000000L + op.FBits * 0x10000000000000L; + + Operand fpScaledMask = X86GetScalar(context, fpScaled); + + nRes = context.AddIntrinsic(Intrinsic.X86Mulsd, nRes, fpScaledMask); + } + + nRes = context.AddIntrinsic(Intrinsic.X86Roundsd, nRes, Const(X86GetRoundControl(roundMode))); + + Operand zero = context.VectorZero(); + + Operand nCmp = context.AddIntrinsic(Intrinsic.X86Cmpsd, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp); + + long fpMaxVal = op.RegisterSize == RegisterSize.Int32 + ? 0x41E0000000000000L // 2147483648.0000000d (2147483648) + : 0x43E0000000000000L; // 9.2233720368547760E18d (9223372036854775808) + + Operand fpMaxValMask = X86GetScalar(context, fpMaxVal); + + Operand nIntOrLong = op.RegisterSize == RegisterSize.Int32 + ? context.AddIntrinsicInt (Intrinsic.X86Cvtsd2si, nRes) + : context.AddIntrinsicLong(Intrinsic.X86Cvtsd2si, nRes); + + nRes = context.AddIntrinsic(Intrinsic.X86Subsd, nRes, fpMaxValMask); + + nCmp = context.AddIntrinsic(Intrinsic.X86Cmpsd, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual)); + nRes = context.AddIntrinsic(Intrinsic.X86Pand, nRes, nCmp); + + Operand nIntOrLong2 = op.RegisterSize == RegisterSize.Int32 + ? context.AddIntrinsicInt (Intrinsic.X86Cvtsd2si, nRes) + : context.AddIntrinsicLong(Intrinsic.X86Cvtsd2si, nRes); + + nRes = context.AddIntrinsic(Intrinsic.X86Cmpsd, nRes, fpMaxValMask, Const((int)CmpCondition.NotLessThan)); + + Operand nLong = context.AddIntrinsicLong(Intrinsic.X86Cvtsi2si, nRes); + + if (op.RegisterSize == RegisterSize.Int32) + { + nLong = context.ConvertI64ToI32(nLong); + } + + Operand dRes = context.BitwiseExclusiveOr(nIntOrLong2, nLong); + dRes = context.Add(dRes, nIntOrLong); + + SetIntOrZR(context, op.Rd, dRes); + } } private static Operand EmitVectorLongExtract(ArmEmitterContext context, int reg, int index, int size) @@ -1145,14 +1479,5 @@ namespace ARMeilleure.Instructions return context.VectorExtract(type, GetVec(reg), index); } - - private static Operand EmitVectorLongCreate(ArmEmitterContext context, Operand low, Operand high) - { - Operand vector = context.VectorCreateScalar(low); - - vector = context.VectorInsert(vector, high, 1); - - return vector; - } } } diff --git a/ARMeilleure/Instructions/InstEmitSimdHelper.cs b/ARMeilleure/Instructions/InstEmitSimdHelper.cs index 28d075dd..fce1bed5 100644 --- a/ARMeilleure/Instructions/InstEmitSimdHelper.cs +++ b/ARMeilleure/Instructions/InstEmitSimdHelper.cs @@ -16,6 +16,24 @@ namespace ARMeilleure.Instructions static class InstEmitSimdHelper { +#region "Masks" + public static readonly long[] EvenMasks = new long[] + { + 14L << 56 | 12L << 48 | 10L << 40 | 08L << 32 | 06L << 24 | 04L << 16 | 02L << 8 | 00L << 0, // B + 13L << 56 | 12L << 48 | 09L << 40 | 08L << 32 | 05L << 24 | 04L << 16 | 01L << 8 | 00L << 0, // H + 11L << 56 | 10L << 48 | 09L << 40 | 08L << 32 | 03L << 24 | 02L << 16 | 01L << 8 | 00L << 0 // S + }; + + public static readonly long[] OddMasks = new long[] + { + 15L << 56 | 13L << 48 | 11L << 40 | 09L << 32 | 07L << 24 | 05L << 16 | 03L << 8 | 01L << 0, // B + 15L << 56 | 14L << 48 | 11L << 40 | 10L << 32 | 07L << 24 | 06L << 16 | 03L << 8 | 02L << 0, // H + 15L << 56 | 14L << 48 | 13L << 40 | 12L << 32 | 07L << 24 | 06L << 16 | 05L << 8 | 04L << 0 // S + }; + + private static readonly long _zeroMask = 128L << 56 | 128L << 48 | 128L << 40 | 128L << 32 | 128L << 24 | 128L << 16 | 128L << 8 | 128L << 0; +#endregion + #region "X86 SSE Intrinsics" public static readonly Intrinsic[] X86PaddInstruction = new Intrinsic[] { @@ -189,11 +207,19 @@ namespace ARMeilleure.Instructions return vector; } + public static Operand X86GetElements(ArmEmitterContext context, long e1, long e0) + { + Operand vector0 = context.VectorCreateScalar(Const(e0)); + Operand vector1 = context.VectorCreateScalar(Const(e1)); + + return context.AddIntrinsic(Intrinsic.X86Punpcklqdq, vector0, vector1); + } + public static int X86GetRoundControl(FPRoundingMode roundMode) { switch (roundMode) { - case FPRoundingMode.ToNearest: return 8 | 0; + case FPRoundingMode.ToNearest: return 8 | 0; // even case FPRoundingMode.TowardsPlusInfinity: return 8 | 2; case FPRoundingMode.TowardsMinusInfinity: return 8 | 1; case FPRoundingMode.TowardsZero: return 8 | 3; @@ -991,6 +1017,46 @@ namespace ARMeilleure.Instructions context.Copy(GetVec(op.Rd), res); } + public static void EmitSsse3VectorPairwiseOp(ArmEmitterContext context, Intrinsic[] inst) + { + OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; + + Operand n = GetVec(op.Rn); + Operand m = GetVec(op.Rm); + + if (op.RegisterSize == RegisterSize.Simd64) + { + Operand zeroEvenMask = X86GetElements(context, _zeroMask, EvenMasks[op.Size]); + Operand zeroOddMask = X86GetElements(context, _zeroMask, OddMasks [op.Size]); + + Operand mN = context.AddIntrinsic(Intrinsic.X86Punpcklqdq, n, m); // m:n + + Operand left = context.AddIntrinsic(Intrinsic.X86Pshufb, mN, zeroEvenMask); // 0:even from m:n + Operand right = context.AddIntrinsic(Intrinsic.X86Pshufb, mN, zeroOddMask); // 0:odd from m:n + + context.Copy(GetVec(op.Rd), context.AddIntrinsic(inst[op.Size], left, right)); + } + else if (op.Size < 3) + { + Operand oddEvenMask = X86GetElements(context, OddMasks[op.Size], EvenMasks[op.Size]); + + Operand oddEvenN = context.AddIntrinsic(Intrinsic.X86Pshufb, n, oddEvenMask); // odd:even from n + Operand oddEvenM = context.AddIntrinsic(Intrinsic.X86Pshufb, m, oddEvenMask); // odd:even from m + + Operand left = context.AddIntrinsic(Intrinsic.X86Punpcklqdq, oddEvenN, oddEvenM); + Operand right = context.AddIntrinsic(Intrinsic.X86Punpckhqdq, oddEvenN, oddEvenM); + + context.Copy(GetVec(op.Rd), context.AddIntrinsic(inst[op.Size], left, right)); + } + else + { + Operand left = context.AddIntrinsic(Intrinsic.X86Punpcklqdq, n, m); + Operand right = context.AddIntrinsic(Intrinsic.X86Punpckhqdq, n, m); + + context.Copy(GetVec(op.Rd), context.AddIntrinsic(inst[3], left, right)); + } + } + public static void EmitVectorAcrossVectorOpSx(ArmEmitterContext context, Func2I emit) { EmitVectorAcrossVectorOp(context, emit, signed: true, isLong: false); @@ -1066,7 +1132,7 @@ namespace ARMeilleure.Instructions context.Copy(GetVec(op.Rd), res); } - public static void EmitVectorPairwiseOpF(ArmEmitterContext context, Intrinsic inst32, Intrinsic inst64) + public static void EmitSse2VectorPairwiseOpF(ArmEmitterContext context, Intrinsic inst32, Intrinsic inst64) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -1231,8 +1297,7 @@ namespace ARMeilleure.Instructions if (op.Size <= 2) { - Operand temp = add ? context.Add (ne, me) - : context.Subtract(ne, me); + Operand temp = add ? context.Add(ne, me) : context.Subtract(ne, me); de = EmitSatQ(context, temp, op.Size, signedSrc: true, signedDst: signed); } @@ -1316,7 +1381,9 @@ namespace ARMeilleure.Instructions int part = !scalar && (op.RegisterSize == RegisterSize.Simd128) ? elems : 0; - Operand res = part == 0 ? context.VectorZero() : context.Copy(GetVec(op.Rd)); + Operand d = GetVec(op.Rd); + + Operand res = part == 0 ? context.VectorZero() : context.Copy(d); for (int index = 0; index < elems; index++) { @@ -1327,7 +1394,7 @@ namespace ARMeilleure.Instructions res = EmitVectorInsert(context, res, temp, part + index, op.Size); } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } // TSrc (16bit, 32bit, 64bit; signed, unsigned) > TDst (8bit, 16bit, 32bit; signed, unsigned). diff --git a/ARMeilleure/Instructions/InstEmitSimdLogical.cs b/ARMeilleure/Instructions/InstEmitSimdLogical.cs index 551752d2..362296f7 100644 --- a/ARMeilleure/Instructions/InstEmitSimdLogical.cs +++ b/ARMeilleure/Instructions/InstEmitSimdLogical.cs @@ -1,6 +1,7 @@ using ARMeilleure.Decoders; using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Translation; +using System.Diagnostics; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; @@ -107,7 +108,7 @@ namespace ARMeilleure.Instructions res = context.VectorZeroUpper64(res); } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } else { @@ -158,7 +159,7 @@ namespace ARMeilleure.Instructions res = context.VectorZeroUpper64(res); } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } else { @@ -292,11 +293,7 @@ namespace ARMeilleure.Instructions { Operand ne = EmitVectorExtractZx(context, op.Rn, index, 0); - ne = context.ConvertI64ToI32(ne); - - Operand de = context.Call(new _U32_U32(SoftFallback.ReverseBits8), ne); - - de = context.ZeroExtend32(OperandType.I64, de); + Operand de = EmitReverseBits8Op(context, ne); res = EmitVectorInsert(context, res, de, index, 0); } @@ -304,6 +301,20 @@ namespace ARMeilleure.Instructions context.Copy(GetVec(op.Rd), res); } + private static Operand EmitReverseBits8Op(ArmEmitterContext context, Operand op) + { + Debug.Assert(op.Type == OperandType.I64); + + Operand val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(op, Const(0xaaul)), Const(1)), + context.ShiftLeft (context.BitwiseAnd(op, Const(0x55ul)), Const(1))); + + val = context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(val, Const(0xccul)), Const(2)), + context.ShiftLeft (context.BitwiseAnd(val, Const(0x33ul)), Const(2))); + + return context.BitwiseOr(context.ShiftRightUI(val, Const(4)), + context.ShiftLeft (context.BitwiseAnd(val, Const(0x0ful)), Const(4))); + } + public static void Rev16_V(ArmEmitterContext context) { if (Optimizations.UseSsse3) diff --git a/ARMeilleure/Instructions/InstEmitSimdMove.cs b/ARMeilleure/Instructions/InstEmitSimdMove.cs index 789c8c87..a1a4635f 100644 --- a/ARMeilleure/Instructions/InstEmitSimdMove.cs +++ b/ARMeilleure/Instructions/InstEmitSimdMove.cs @@ -13,20 +13,6 @@ namespace ARMeilleure.Instructions static partial class InstEmit { #region "Masks" - private static readonly long[] _masksE0_TrnUzpXtn = new long[] - { - 14L << 56 | 12L << 48 | 10L << 40 | 08L << 32 | 06L << 24 | 04L << 16 | 02L << 8 | 00L << 0, - 13L << 56 | 12L << 48 | 09L << 40 | 08L << 32 | 05L << 24 | 04L << 16 | 01L << 8 | 00L << 0, - 11L << 56 | 10L << 48 | 09L << 40 | 08L << 32 | 03L << 24 | 02L << 16 | 01L << 8 | 00L << 0 - }; - - private static readonly long[] _masksE1_TrnUzp = new long[] - { - 15L << 56 | 13L << 48 | 11L << 40 | 09L << 32 | 07L << 24 | 05L << 16 | 03L << 8 | 01L << 0, - 15L << 56 | 14L << 48 | 11L << 40 | 10L << 32 | 07L << 24 | 06L << 16 | 03L << 8 | 02L << 0, - 15L << 56 | 14L << 48 | 13L << 40 | 12L << 32 | 07L << 24 | 06L << 16 | 05L << 8 | 04L << 0 - }; - private static readonly long[] _masksE0_Uzp = new long[] { 13L << 56 | 09L << 48 | 05L << 40 | 01L << 32 | 12L << 24 | 08L << 16 | 04L << 8 | 00L << 0, @@ -447,7 +433,7 @@ namespace ARMeilleure.Instructions Operand res = context.VectorZeroUpper64(d); - Operand mask = X86GetAllElements(context, _masksE0_TrnUzpXtn[op.Size]); + Operand mask = X86GetAllElements(context, EvenMasks[op.Size]); Operand res2 = context.AddIntrinsic(Intrinsic.X86Pshufb, GetVec(op.Rn), mask); @@ -646,8 +632,8 @@ namespace ARMeilleure.Instructions if (op.Size < 3) { - long maskE0 = _masksE0_TrnUzpXtn[op.Size]; - long maskE1 = _masksE1_TrnUzp [op.Size]; + long maskE0 = EvenMasks[op.Size]; + long maskE1 = OddMasks [op.Size]; mask = X86GetScalar(context, maskE0); @@ -714,8 +700,8 @@ namespace ARMeilleure.Instructions if (op.Size < 3) { - long maskE0 = _masksE0_TrnUzpXtn[op.Size]; - long maskE1 = _masksE1_TrnUzp [op.Size]; + long maskE0 = EvenMasks[op.Size]; + long maskE1 = OddMasks [op.Size]; mask = X86GetScalar(context, maskE0); diff --git a/ARMeilleure/Instructions/InstEmitSimdShift.cs b/ARMeilleure/Instructions/InstEmitSimdShift.cs index 17f43c81..19c0a74d 100644 --- a/ARMeilleure/Instructions/InstEmitSimdShift.cs +++ b/ARMeilleure/Instructions/InstEmitSimdShift.cs @@ -4,6 +4,7 @@ using ARMeilleure.Decoders; using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Translation; using System; +using System.Diagnostics; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; @@ -16,13 +17,6 @@ namespace ARMeilleure.Instructions static partial class InstEmit { #region "Masks" - private static readonly long[] _masks_RshrnShrn = new long[] - { - 14L << 56 | 12L << 48 | 10L << 40 | 08L << 32 | 06L << 24 | 04L << 16 | 02L << 8 | 00L << 0, - 13L << 56 | 12L << 48 | 09L << 40 | 08L << 32 | 05L << 24 | 04L << 16 | 01L << 8 | 00L << 0, - 11L << 56 | 10L << 48 | 09L << 40 | 08L << 32 | 03L << 24 | 02L << 16 | 01L << 8 | 00L << 0 - }; - private static readonly long[] _masks_SliSri = new long[] // Replication masks. { 0x0101010101010101L, 0x0001000100010001L, 0x0000000100000001L, 0x0000000000000001L @@ -42,7 +36,7 @@ namespace ARMeilleure.Instructions Operand d = GetVec(op.Rd); Operand n = GetVec(op.Rn); - Operand dLow = context.AddIntrinsic(Intrinsic.X86Movlhps, d, context.VectorZero()); + Operand dLow = context.VectorZeroUpper64(d); Operand mask = null; @@ -61,7 +55,7 @@ namespace ARMeilleure.Instructions res = context.AddIntrinsic(srlInst, res, Const(shift)); - Operand mask2 = X86GetAllElements(context, _masks_RshrnShrn[op.Size]); + Operand mask2 = X86GetAllElements(context, EvenMasks[op.Size]); res = context.AddIntrinsic(Intrinsic.X86Pshufb, res, mask2); @@ -157,13 +151,13 @@ namespace ARMeilleure.Instructions Operand d = GetVec(op.Rd); Operand n = GetVec(op.Rn); - Operand dLow = context.AddIntrinsic(Intrinsic.X86Movlhps, d, context.VectorZero()); + Operand dLow = context.VectorZeroUpper64(d); Intrinsic srlInst = X86PsrlInstruction[op.Size + 1]; Operand nShifted = context.AddIntrinsic(srlInst, n, Const(shift)); - Operand mask = X86GetAllElements(context, _masks_RshrnShrn[op.Size]); + Operand mask = X86GetAllElements(context, EvenMasks[op.Size]); Operand res = context.AddIntrinsic(Intrinsic.X86Pshufb, nShifted, mask); @@ -702,9 +696,9 @@ namespace ARMeilleure.Instructions for (int index = 0; index < elems; index++) { Operand ne = EmitVectorExtractZx(context, op.Rn, index, op.Size); - Operand me = EmitVectorExtractZx(context, op.Rm, index, op.Size); + Operand me = EmitVectorExtractSx(context, op.Rm, index << op.Size, 0); - Operand e = context.Call(new _U64_U64_U64_Bool_S32(SoftFallback.UnsignedShlReg), ne, me, Const(0), Const(op.Size)); + Operand e = EmitUnsignedShlRegOp(context, ne, context.ConvertI64ToI32(me), op.Size); res = EmitVectorInsert(context, res, e, index, op.Size); } @@ -879,9 +873,7 @@ namespace ARMeilleure.Instructions e = context.Add(e, Const(roundConst)); } - e = signed - ? context.ShiftRightSI(e, Const(shift)) - : context.ShiftRightUI(e, Const(shift)); + e = signed ? context.ShiftRightSI(e, Const(shift)) : context.ShiftRightUI(e, Const(shift)); } else /* if (op.Size == 3) */ { @@ -901,6 +893,28 @@ namespace ARMeilleure.Instructions context.Copy(GetVec(op.Rd), res); } + private static Operand EmitUnsignedShlRegOp(ArmEmitterContext context, Operand op, Operand shiftLsB, int size) + { + Debug.Assert(op.Type == OperandType.I64); + Debug.Assert(shiftLsB.Type == OperandType.I32); + Debug.Assert((uint)size < 4u); + + Operand negShiftLsB = context.Negate(shiftLsB); + + Operand isPositive = context.ICompareGreaterOrEqual(shiftLsB, Const(0)); + + Operand shl = context.ShiftLeft (op, shiftLsB); + Operand shr = context.ShiftRightUI(op, negShiftLsB); + + Operand res = context.ConditionalSelect(isPositive, shl, shr); + + Operand isOutOfRange = context.BitwiseOr( + context.ICompareGreaterOrEqual(shiftLsB, Const(8 << size)), + context.ICompareGreaterOrEqual(negShiftLsB, Const(8 << size))); + + return context.ConditionalSelect(isOutOfRange, Const(0UL), res); + } + private static void EmitVectorShrImmNarrowOpZx(ArmEmitterContext context, bool round) { OpCodeSimdShImm op = (OpCodeSimdShImm)context.CurrOp; @@ -913,7 +927,9 @@ namespace ARMeilleure.Instructions int part = op.RegisterSize == RegisterSize.Simd128 ? elems : 0; - Operand res = part == 0 ? context.VectorZero() : context.Copy(GetVec(op.Rd)); + Operand d = GetVec(op.Rd); + + Operand res = part == 0 ? context.VectorZero() : context.Copy(d); for (int index = 0; index < elems; index++) { @@ -929,7 +945,7 @@ namespace ARMeilleure.Instructions res = EmitVectorInsert(context, res, e, part + index, op.Size); } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } [Flags] @@ -972,7 +988,9 @@ namespace ARMeilleure.Instructions int part = !scalar && (op.RegisterSize == RegisterSize.Simd128) ? elems : 0; - Operand res = part == 0 ? context.VectorZero() : context.Copy(GetVec(op.Rd)); + Operand d = GetVec(op.Rd); + + Operand res = part == 0 ? context.VectorZero() : context.Copy(d); for (int index = 0; index < elems; index++) { @@ -985,9 +1003,7 @@ namespace ARMeilleure.Instructions e = context.Add(e, Const(roundConst)); } - e = signedSrc - ? context.ShiftRightSI(e, Const(shift)) - : context.ShiftRightUI(e, Const(shift)); + e = signedSrc ? context.ShiftRightSI(e, Const(shift)) : context.ShiftRightUI(e, Const(shift)); } else /* if (op.Size == 2 && round) */ { @@ -999,7 +1015,7 @@ namespace ARMeilleure.Instructions res = EmitVectorInsert(context, res, e, part + index, op.Size); } - context.Copy(GetVec(op.Rd), res); + context.Copy(d, res); } // dst64 = (Int(src64, signed) + roundConst) >> shift; diff --git a/ARMeilleure/Instructions/SoftFallback.cs b/ARMeilleure/Instructions/SoftFallback.cs index 8d48c34f..10bb47df 100644 --- a/ARMeilleure/Instructions/SoftFallback.cs +++ b/ARMeilleure/Instructions/SoftFallback.cs @@ -1240,74 +1240,5 @@ namespace ARMeilleure.Instructions : (uint)(value >> 32); } #endregion - -#region "Reverse" - public static uint ReverseBits8(uint value) - { - value = ((value & 0xaa) >> 1) | ((value & 0x55) << 1); - value = ((value & 0xcc) >> 2) | ((value & 0x33) << 2); - - return (value >> 4) | ((value & 0x0f) << 4); - } - - public static uint ReverseBits32(uint value) - { - value = ((value & 0xaaaaaaaa) >> 1) | ((value & 0x55555555) << 1); - value = ((value & 0xcccccccc) >> 2) | ((value & 0x33333333) << 2); - value = ((value & 0xf0f0f0f0) >> 4) | ((value & 0x0f0f0f0f) << 4); - value = ((value & 0xff00ff00) >> 8) | ((value & 0x00ff00ff) << 8); - - return (value >> 16) | (value << 16); - } - - public static ulong ReverseBits64(ulong value) - { - value = ((value & 0xaaaaaaaaaaaaaaaa) >> 1 ) | ((value & 0x5555555555555555) << 1 ); - value = ((value & 0xcccccccccccccccc) >> 2 ) | ((value & 0x3333333333333333) << 2 ); - value = ((value & 0xf0f0f0f0f0f0f0f0) >> 4 ) | ((value & 0x0f0f0f0f0f0f0f0f) << 4 ); - value = ((value & 0xff00ff00ff00ff00) >> 8 ) | ((value & 0x00ff00ff00ff00ff) << 8 ); - value = ((value & 0xffff0000ffff0000) >> 16) | ((value & 0x0000ffff0000ffff) << 16); - - return (value >> 32) | (value << 32); - } - - public static uint ReverseBytes16_32(uint value) => (uint)ReverseBytes16_64(value); - - public static ulong ReverseBytes16_64(ulong value) => ReverseBytes(value, RevSize.Rev16); - public static ulong ReverseBytes32_64(ulong value) => ReverseBytes(value, RevSize.Rev32); - - private enum RevSize - { - Rev16, - Rev32, - Rev64 - } - - private static ulong ReverseBytes(ulong value, RevSize size) - { - value = ((value & 0xff00ff00ff00ff00) >> 8) | ((value & 0x00ff00ff00ff00ff) << 8); - - if (size == RevSize.Rev16) - { - return value; - } - - value = ((value & 0xffff0000ffff0000) >> 16) | ((value & 0x0000ffff0000ffff) << 16); - - if (size == RevSize.Rev32) - { - return value; - } - - value = ((value & 0xffffffff00000000) >> 32) | ((value & 0x00000000ffffffff) << 32); - - if (size == RevSize.Rev64) - { - return value; - } - - throw new ArgumentException(nameof(size)); - } -#endregion } } diff --git a/ARMeilleure/IntermediateRepresentation/Intrinsic.cs b/ARMeilleure/IntermediateRepresentation/Intrinsic.cs index 57c8914d..c3f375c4 100644 --- a/ARMeilleure/IntermediateRepresentation/Intrinsic.cs +++ b/ARMeilleure/IntermediateRepresentation/Intrinsic.cs @@ -30,8 +30,11 @@ namespace ARMeilleure.IntermediateRepresentation X86Cvtps2pd, X86Cvtsd2si, X86Cvtsd2ss, + X86Cvtsi2sd, X86Cvtsi2si, + X86Cvtsi2ss, X86Cvtss2sd, + X86Cvtss2si, X86Divpd, X86Divps, X86Divsd, diff --git a/Ryujinx.Tests/Cpu/CpuTestSimd.cs b/Ryujinx.Tests/Cpu/CpuTestSimd.cs index 30dec59a..8f7e2069 100644 --- a/Ryujinx.Tests/Cpu/CpuTestSimd.cs +++ b/Ryujinx.Tests/Cpu/CpuTestSimd.cs @@ -2856,7 +2856,7 @@ namespace Ryujinx.Tests.Cpu SingleOpcode(opcodes, v0: v0, v1: v1); - CompareAgainstUnicorn(fpTolerances: FpTolerances.UpToOneUlpsD); // unsigned + CompareAgainstUnicorn(); } [Test, Pairwise] [Explicit] @@ -2892,7 +2892,7 @@ namespace Ryujinx.Tests.Cpu SingleOpcode(opcodes, v0: v0, v1: v1); - CompareAgainstUnicorn(fpTolerances: FpTolerances.UpToOneUlpsD); // unsigned + CompareAgainstUnicorn(); } [Test, Pairwise] diff --git a/Ryujinx.Tests/Cpu/CpuTestSimdCvt.cs b/Ryujinx.Tests/Cpu/CpuTestSimdCvt.cs index 17a2853f..2d5c8231 100644 --- a/Ryujinx.Tests/Cpu/CpuTestSimdCvt.cs +++ b/Ryujinx.Tests/Cpu/CpuTestSimdCvt.cs @@ -582,7 +582,7 @@ namespace Ryujinx.Tests.Cpu SingleOpcode(opcodes, x1: xn, x31: x31, v0: v0); - CompareAgainstUnicorn(fpTolerances: FpTolerances.UpToOneUlpsD); // unsigned + CompareAgainstUnicorn(); } [Test, Pairwise] [Explicit] @@ -666,7 +666,7 @@ namespace Ryujinx.Tests.Cpu SingleOpcode(opcodes, x1: xn, x31: x31, v0: v0); - CompareAgainstUnicorn(fpTolerances: FpTolerances.UpToOneUlpsD); // unsigned + CompareAgainstUnicorn(); } #endif } diff --git a/Ryujinx.Tests/Cpu/CpuTestSimdShImm.cs b/Ryujinx.Tests/Cpu/CpuTestSimdShImm.cs index 0f1b0dac..1d208d69 100644 --- a/Ryujinx.Tests/Cpu/CpuTestSimdShImm.cs +++ b/Ryujinx.Tests/Cpu/CpuTestSimdShImm.cs @@ -564,7 +564,7 @@ namespace Ryujinx.Tests.Cpu SingleOpcode(opcodes, v0: v0, v1: v1); - CompareAgainstUnicorn(fpTolerances: FpTolerances.UpToOneUlpsD); // unsigned + CompareAgainstUnicorn(); } [Test, Pairwise]