Reduce JIT GC allocations (#2515)

* Turn `MemoryOperand` into a struct

* Remove `IntrinsicOperation`

* Remove `PhiNode`

* Remove `Node`

* Turn `Operand` into a struct

* Turn `Operation` into a struct

* Clean up pool management methods

* Add `Arena` allocator

* Move `OperationHelper` to `Operation.Factory`

* Move `OperandHelper` to `Operand.Factory`

* Optimize `Operation` a bit

* Fix `Arena` initialization

* Rename `NativeList<T>` to `ArenaList<T>`

* Reduce `Operand` size from 88 to 56 bytes

* Reduce `Operation` size from 56 to 40 bytes

* Add optimistic interning of Register & Constant operands

* Optimize `RegisterUsage` pass a bit

* Optimize `RemoveUnusedNodes` pass a bit

Iterating in reverse-order allows killing dependency chains in a single
pass.

* Fix PPTC symbols

* Optimize `BasicBlock` a bit

Reduce allocations from `_successor` & `DominanceFrontiers`

* Fix `Operation` resize

* Make `Arena` expandable

Change the arena allocator to be expandable by allocating in pages, with
some of them being pooled. Currently 32 pages are pooled. An LRU removal
mechanism should probably be added to it.

Apparently MHR can allocate bitmaps large enough to exceed the 16MB
limit for the type.

* Move `Arena` & `ArenaList` to `Common`

* Remove `ThreadStaticPool` & co

* Add `PhiOperation`

* Reduce `Operand` size from 56 from 48 bytes

* Add linear-probing to `Operand` intern table

* Optimize `HybridAllocator` a bit

* Add `Allocators` class

* Tune `ArenaAllocator` sizes

* Add page removal mechanism to `ArenaAllocator`

Remove pages which have not been used for more than 5s after each reset.

I am on fence if this would be better using a Gen2 callback object like
the one in System.Buffers.ArrayPool<T>, to trim the pool. Because right
now if a large translation happens, the pages will be freed only after a
reset. This reset may not happen for a while because no new translation
is hit, but the arena base sizes are rather small.

* Fix `OOM` when allocating larger than page size in `ArenaAllocator`

Tweak resizing mechanism for Operand.Uses and Assignemnts.

* Optimize `Optimizer` a bit

* Optimize `Operand.Add<T>/Remove<T>` a bit

* Clean up `PreAllocator`

* Fix phi insertion order

Reduce codegen diffs.

* Fix code alignment

* Use new heuristics for degree of parallelism

* Suppress warnings

* Address gdkchan's feedback

Renamed `GetValue()` to `GetValueUnsafe()` to make it more clear that
`Operand.Value` should usually not be modified directly.

* Add fast path to `ArenaAllocator`

* Assembly for `ArenaAllocator.Allocate(ulong)`:

  .L0:
    mov rax, [rcx+0x18]
    lea r8, [rax+rdx]
    cmp r8, [rcx+0x10]
    ja short .L2
  .L1:
    mov rdx, [rcx+8]
    add rax, [rdx+8]
    mov [rcx+0x18], r8
    ret
  .L2:
    jmp ArenaAllocator.AllocateSlow(UInt64)

  A few variable/field had to be changed to ulong so that RyuJIT avoids
  emitting zero-extends.

* Implement a new heuristic to free pooled pages.

  If an arena is used often, it is more likely that its pages will be
  needed, so the pages are kept for longer (e.g: during PPTC rebuild or
  burst sof compilations). If is not used often, then it is more likely
  that its pages will not be needed (e.g: after PPTC rebuild or bursts
  of compilations).

* Address riperiperi's feedback

* Use `EqualityComparer<T>` in `IntrusiveList<T>`

Avoids a potential GC hole in `Equals(T, T)`.

ARMeilleure/Allocators.cs

@@ -0,0 +1,38 @@
+using ARMeilleure.Common;
+using System;
+using System.Runtime.CompilerServices;
+
+namespace ARMeilleure
+{
+    static class Allocators
+    {
+        [ThreadStatic] private static ArenaAllocator _default;
+        [ThreadStatic] private static ArenaAllocator _operands;
+        [ThreadStatic] private static ArenaAllocator _operations;
+        [ThreadStatic] private static ArenaAllocator _references;
+
+        public static ArenaAllocator Default => GetAllocator(ref _default, 256 * 1024, 4);
+        public static ArenaAllocator Operands => GetAllocator(ref _operands, 64 * 1024, 8);
+        public static ArenaAllocator Operations => GetAllocator(ref _operations, 64 * 1024, 8);
+        public static ArenaAllocator References => GetAllocator(ref _references, 64 * 1024, 8);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static ArenaAllocator GetAllocator(ref ArenaAllocator alloc, uint pageSize, uint pageCount)
+        {
+            if (alloc == null)
+            {
+                alloc = new ArenaAllocator(pageSize, pageCount);
+            }
+
+            return alloc;
+        }
+
+        public static void ResetAll()
+        {
+            Default.Reset();
+            Operands.Reset();
+            Operations.Reset();
+            References.Reset();
+        }
+    }
+}

ARMeilleure/CodeGen/Optimizations/BlockPlacement.cs

@@ -1,8 +1,7 @@
 using ARMeilleure.IntermediateRepresentation;
 using ARMeilleure.Translation;
 using System.Diagnostics;
-
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
 
 namespace ARMeilleure.CodeGen.Optimizations
 {
@@ -33,8 +32,10 @@ namespace ARMeilleure.CodeGen.Optimizations
             {
                 nextBlock = block.ListNext;
 
-                if (block.SuccessorCount == 2 && block.Operations.Last is Operation branchOp)
+                if (block.SuccessorsCount == 2)
                 {
+                    Operation branchOp = block.Operations.Last;
+
                     Debug.Assert(branchOp.Instruction == Instruction.BranchIf);
 
                     BasicBlock falseSucc = block.GetSuccessor(0);
@@ -59,7 +60,7 @@ namespace ARMeilleure.CodeGen.Optimizations
 
             if (update)
             {
-                cfg.Update(removeUnreachableBlocks: false);
+                cfg.Update();
             }
         }
     }

ARMeilleure/CodeGen/Optimizations/ConstantFolding.cs

@@ -1,7 +1,6 @@
 using ARMeilleure.IntermediateRepresentation;
 using System;
-
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
 
 namespace ARMeilleure.CodeGen.Optimizations
 {
@@ -9,7 +8,7 @@ namespace ARMeilleure.CodeGen.Optimizations
     {
         public static void RunPass(Operation operation)
         {
-            if (operation.Destination == null || operation.SourcesCount == 0)
+            if (operation.Destination == default || operation.SourcesCount == 0)
             {
                 return;
             }

ARMeilleure/CodeGen/Optimizations/Optimizer.cs

@@ -1,8 +1,8 @@
 using ARMeilleure.IntermediateRepresentation;
 using ARMeilleure.Translation;
+using System;
 using System.Diagnostics;
-
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
 
 namespace ARMeilleure.CodeGen.Optimizations
 {
@@ -10,62 +10,60 @@ namespace ARMeilleure.CodeGen.Optimizations
     {
         public static void RunPass(ControlFlowGraph cfg)
         {
+            // Scratch buffer used to store uses.
+            Span<Operation> buffer = default;
+
             bool modified;
 
             do
             {
                 modified = false;
 
-                for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext)
+                for (BasicBlock block = cfg.Blocks.Last; block != null; block = block.ListPrevious)
                 {
-                    Node node = block.Operations.First;
+                    Operation node;
+                    Operation prevNode;
 
-                    while (node != null)
+                    for (node = block.Operations.Last; node != default; node = prevNode)
                     {
-                        Node nextNode = node.ListNext;
+                        prevNode = node.ListPrevious;
 
-                        bool isUnused = IsUnused(node);
+                        if (IsUnused(node))
-
-                        if (!(node is Operation operation) || isUnused)
-                        {
-                            if (isUnused)
                         {
                             RemoveNode(block, node);
 
                             modified = true;
-                            }
-
-                            node = nextNode;
 
                             continue;
                         }
-
+                        else if (node.Instruction == Instruction.Phi)
-                        ConstantFolding.RunPass(operation);
-
-                        Simplification.RunPass(operation);
-
-                        if (DestIsLocalVar(operation))
                         {
-                            if (IsPropagableCompare(operation))
+                            continue;
-                            {
+                        }
-                                modified |= PropagateCompare(operation);
 
-                                if (modified && IsUnused(operation))
+                        ConstantFolding.RunPass(node);
+                        Simplification.RunPass(node);
+
+                        if (DestIsLocalVar(node))
+                        {   
+                            if (IsPropagableCompare(node))
+                            {
+                                modified |= PropagateCompare(ref buffer, node);
+
+                                if (modified && IsUnused(node))
                                 {
                                     RemoveNode(block, node);
                                 }
                             }
-                            else if (IsPropagableCopy(operation))
+                            else if (IsPropagableCopy(node))
                             {
-                                PropagateCopy(operation);
+                                PropagateCopy(ref buffer, node);
 
                                 RemoveNode(block, node);
 
                                 modified = true;
                             }
                         }
-
-                        node = nextNode;
                     }
                 }
             }
@@ -80,13 +78,14 @@ namespace ARMeilleure.CodeGen.Optimizations
             {
                 modified = false;
 
-                for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext)
+                for (BasicBlock block = cfg.Blocks.Last; block != null; block = block.ListPrevious)
                 {
-                    Node node = block.Operations.First;
+                    Operation node;
+                    Operation prevNode;
 
-                    while (node != null)
+                    for (node = block.Operations.Last; node != default; node = prevNode)
                     {
-                        Node nextNode = node.ListNext;
+                        prevNode = node.ListPrevious;
 
                         if (IsUnused(node))
                         {
@@ -94,15 +93,27 @@ namespace ARMeilleure.CodeGen.Optimizations
 
                             modified = true;
                         }
-
-                        node = nextNode;
                     }
                 }
             }
             while (modified);
         }
 
-        private static bool PropagateCompare(Operation compOp)
+        private static Span<Operation> GetUses(ref Span<Operation> buffer, Operand operand)
+        {
+            ReadOnlySpan<Operation> uses = operand.Uses;
+
+            if (buffer.Length < uses.Length)
+            {
+                buffer = Allocators.Default.AllocateSpan<Operation>((uint)uses.Length);
+            }
+
+            uses.CopyTo(buffer);
+
+            return buffer.Slice(0, uses.Length);
+        }
+
+        private static bool PropagateCompare(ref Span<Operation> buffer, Operation compOp)
         {
             // Try to propagate Compare operations into their BranchIf uses, when these BranchIf uses are in the form
             // of:
@@ -149,17 +160,12 @@ namespace ARMeilleure.CodeGen.Optimizations
 
             Comparison compType = (Comparison)comp.AsInt32();
 
-            Node[] uses = dest.Uses.ToArray();
+            Span<Operation> uses = GetUses(ref buffer, dest);
 
-            foreach (Node use in uses)
+            foreach (Operation use in uses)
             {
-                if (!(use is Operation operation))
-                {
-                    continue;
-                }
-
                 // If operation is a BranchIf and has a constant value 0 in its RHS or LHS source operands.
-                if (IsZeroBranch(operation, out Comparison otherCompType))
+                if (IsZeroBranch(use, out Comparison otherCompType))
                 {
                     Comparison propCompType;
 
@@ -176,9 +182,9 @@ namespace ARMeilleure.CodeGen.Optimizations
                         continue;
                     }
 
-                    operation.SetSource(0, src1);
+                    use.SetSource(0, src1);
-                    operation.SetSource(1, src2);
+                    use.SetSource(1, src2);
-                    operation.SetSource(2, Const((int)propCompType));
+                    use.SetSource(2, Const((int)propCompType));
 
                     modified = true;
                 }
@@ -187,15 +193,15 @@ namespace ARMeilleure.CodeGen.Optimizations
             return modified;
         }
 
-        private static void PropagateCopy(Operation copyOp)
+        private static void PropagateCopy(ref Span<Operation> buffer, Operation copyOp)
         {
             // Propagate copy source operand to all uses of the destination operand.
             Operand dest   = copyOp.Destination;
             Operand source = copyOp.GetSource(0);
 
-            Node[] uses = dest.Uses.ToArray();
+            Span<Operation> uses = GetUses(ref buffer, dest);
 
-            foreach (Node use in uses)
+            foreach (Operation use in uses)
             {
                 for (int index = 0; index < use.SourcesCount; index++)
                 {
@@ -207,7 +213,7 @@ namespace ARMeilleure.CodeGen.Optimizations
             }
         }
 
-        private static void RemoveNode(BasicBlock block, Node node)
+        private static void RemoveNode(BasicBlock block, Operation node)
         {
             // Remove a node from the nodes list, and also remove itself
             // from all the use lists on the operands that this node uses.
@@ -215,31 +221,31 @@ namespace ARMeilleure.CodeGen.Optimizations
 
             for (int index = 0; index < node.SourcesCount; index++)
             {
-                node.SetSource(index, null);
+                node.SetSource(index, default);
             }
 
-            Debug.Assert(node.Destination == null || node.Destination.Uses.Count == 0);
+            Debug.Assert(node.Destination == default || node.Destination.UsesCount == 0);
 
-            node.Destination = null;
+            node.Destination = default;
         }
 
-        private static bool IsUnused(Node node)
+        private static bool IsUnused(Operation node)
         {
-            return DestIsLocalVar(node) && node.Destination.Uses.Count == 0 && !HasSideEffects(node);
+            return DestIsLocalVar(node) && node.Destination.UsesCount == 0 && !HasSideEffects(node);
         }
 
-        private static bool DestIsLocalVar(Node node)
+        private static bool DestIsLocalVar(Operation node)
         {
-            return node.Destination != null && node.Destination.Kind == OperandKind.LocalVariable;
+            return node.Destination != default && node.Destination.Kind == OperandKind.LocalVariable;
         }
 
-        private static bool HasSideEffects(Node node)
+        private static bool HasSideEffects(Operation node)
         {
-            return (node is Operation operation) && (operation.Instruction == Instruction.Call
+            return node.Instruction == Instruction.Call
-                || operation.Instruction == Instruction.Tailcall
+                || node.Instruction == Instruction.Tailcall
-                || operation.Instruction == Instruction.CompareAndSwap
+                || node.Instruction == Instruction.CompareAndSwap
-                || operation.Instruction == Instruction.CompareAndSwap16
+                || node.Instruction == Instruction.CompareAndSwap16
-                || operation.Instruction == Instruction.CompareAndSwap8);
+                || node.Instruction == Instruction.CompareAndSwap8;
         }
 
         private static bool IsPropagableCompare(Operation operation)

ARMeilleure/CodeGen/Optimizations/Simplification.cs

@@ -1,7 +1,6 @@
 using ARMeilleure.IntermediateRepresentation;
 using System;
-
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
 
 namespace ARMeilleure.CodeGen.Optimizations
 {

ARMeilleure/CodeGen/RegisterAllocators/CopyResolver.cs

@@ -1,9 +1,8 @@
 using ARMeilleure.IntermediateRepresentation;
 using System;
 using System.Collections.Generic;
-
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operation.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperationHelper;
 
 namespace ARMeilleure.CodeGen.RegisterAllocators
 {
@@ -210,7 +209,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
 
             Operand register = GetRegister(left.Register, type);
 
-            _spillQueue.Enqueue(Operation(Instruction.Spill, null, offset, register));
+            _spillQueue.Enqueue(Operation(Instruction.Spill, default, offset, register));
 
             HasCopy = true;
         }

ARMeilleure/CodeGen/RegisterAllocators/HybridAllocator.cs

@@ -1,11 +1,10 @@
 using ARMeilleure.IntermediateRepresentation;
 using ARMeilleure.Translation;
-using System.Collections.Generic;
+using System;
 using System.Diagnostics;
 using System.Numerics;
-
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operation.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperationHelper;
 
 namespace ARMeilleure.CodeGen.RegisterAllocators
 {
@@ -29,19 +28,14 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             }
         }
 
-        private class LocalInfo
+        private struct LocalInfo
         {
             public int Uses { get; set; }
-            public int UseCount { get; set; }
+            public int UsesAllocated { get; set; }
-
-            public bool PreAllocated { get; set; }
             public int Register { get; set; }
             public int SpillOffset { get; set; }
-
             public int Sequence { get; set; }
-
             public Operand Temp { get; set; }
-
             public OperandType Type { get; }
 
             private int _first;
@@ -49,13 +43,21 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
 
             public bool IsBlockLocal => _first == _last;
 
-            public LocalInfo(OperandType type, int uses)
+            public LocalInfo(OperandType type, int uses, int blkIndex)
             {
                 Uses = uses;
                 Type = type;
 
+                UsesAllocated = 0;
+                Register = 0;
+                SpillOffset = 0;
+                Sequence = 0;
+                Temp = default;
+
                 _first = -1;
                 _last  = -1;
+
+                SetBlockIndex(blkIndex);
             }
 
             public void SetBlockIndex(int blkIndex)
@@ -72,10 +74,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             }
         }
 
-        public AllocationResult RunPass(
+        public AllocationResult RunPass(ControlFlowGraph cfg, StackAllocator stackAlloc, RegisterMasks regMasks)
-            ControlFlowGraph cfg,
-            StackAllocator stackAlloc,
-            RegisterMasks regMasks)
         {
             int intUsedRegisters = 0;
             int vecUsedRegisters = 0;
@@ -84,9 +83,33 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             int vecFreeRegisters = regMasks.VecAvailableRegisters;
 
             var blockInfo = new BlockInfo[cfg.Blocks.Count];
+            var localInfo = new LocalInfo[cfg.Blocks.Count * 3];
+            int localInfoCount = 0;
 
-            var locInfo = new List<LocalInfo>();
+            // The "visited" state is stored in the MSB of the local's value.
-            var locVisited = new HashSet<Operand>();
+            const ulong VisitedMask = 1ul << 63;
+
+            bool IsVisited(Operand local)
+            {
+                return (local.GetValueUnsafe() & VisitedMask) != 0;
+            }
+
+            void SetVisited(Operand local)
+            {
+                local.GetValueUnsafe() |= VisitedMask | (uint)++localInfoCount;
+            }
+
+            ref LocalInfo GetLocalInfo(Operand local)
+            {
+                Debug.Assert(local.Kind == OperandKind.LocalVariable);
+
+                if (!IsVisited(local))
+                {
+                    throw new InvalidOperationException("Local was not visisted yet. Used before defined?");
+                }
+
+                return ref localInfo[(uint)local.GetValueUnsafe() - 1];
+            }
 
             for (int index = cfg.PostOrderBlocks.Length - 1; index >= 0; index--)
             {
@@ -97,59 +120,58 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
 
                 bool hasCall = false;
 
-                for (Node node = block.Operations.First; node != null; node = node.ListNext)
+                for (Operation node = block.Operations.First; node != default; node = node.ListNext)
                 {
-                    if (node is Operation operation && operation.Instruction == Instruction.Call)
+                    if (node.Instruction == Instruction.Call)
                     {
                         hasCall = true;
                     }
 
-                    for (int srcIndex = 0; srcIndex < node.SourcesCount; srcIndex++)
+                    for (int i = 0; i < node.SourcesCount; i++)
                     {
-                        Operand source = node.GetSource(srcIndex);
+                        Operand source = node.GetSource(i);
 
                         if (source.Kind == OperandKind.LocalVariable)
                         {
-                            locInfo[source.GetLocalNumber() - 1].SetBlockIndex(block.Index);
+                            GetLocalInfo(source).SetBlockIndex(block.Index);
                         }
                         else if (source.Kind == OperandKind.Memory)
                         {
-                            MemoryOperand memOp = (MemoryOperand)source;
+                            MemoryOperand memOp = source.GetMemory();
 
-                            if (memOp.BaseAddress != null)
+                            if (memOp.BaseAddress != default)
                             {
-                                locInfo[memOp.BaseAddress.GetLocalNumber() - 1].SetBlockIndex(block.Index);
+                                GetLocalInfo(memOp.BaseAddress).SetBlockIndex(block.Index);
                             }
 
-                            if (memOp.Index != null)
+                            if (memOp.Index != default)
                             {
-                                locInfo[memOp.Index.GetLocalNumber() - 1].SetBlockIndex(block.Index);
+                                GetLocalInfo(memOp.Index).SetBlockIndex(block.Index);
                             }
                         }
                     }
 
-                    for (int dstIndex = 0; dstIndex < node.DestinationsCount; dstIndex++)
+                    for (int i = 0; i < node.DestinationsCount; i++)
                     {
-                        Operand dest = node.GetDestination(dstIndex);
+                        Operand dest = node.GetDestination(i);
 
                         if (dest.Kind == OperandKind.LocalVariable)
                         {
-                            LocalInfo info;
+                            if (IsVisited(dest))
-
-                            if (!locVisited.Add(dest))
                             {
-                                info = locInfo[dest.GetLocalNumber() - 1];
+                                GetLocalInfo(dest).SetBlockIndex(block.Index);
                             }
                             else
                             {
-                                dest.NumberLocal(locInfo.Count + 1);
+                                SetVisited(dest);
 
-                                info = new LocalInfo(dest.Type, UsesCount(dest));
+                                if (localInfoCount > localInfo.Length)
-
+                                {
-                                locInfo.Add(info);
+                                    Array.Resize(ref localInfo, localInfoCount * 2);
                                 }
 
-                            info.SetBlockIndex(block.Index);
+                                GetLocalInfo(dest) = new LocalInfo(dest.Type, UsesCount(dest), block.Index);
+                            }
                         }
                         else if (dest.Kind == OperandKind.Register)
                         {
@@ -192,42 +214,26 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                 intLocalFreeRegisters &= ~(intSpillTempRegisters | intCallerSavedRegisters);
                 vecLocalFreeRegisters &= ~(vecSpillTempRegisters | vecCallerSavedRegisters);
 
-                for (Node node = block.Operations.First; node != null; node = node.ListNext)
+                for (Operation node = block.Operations.First; node != default; node = node.ListNext)
                 {
                     int intLocalUse = 0;
                     int vecLocalUse = 0;
 
-                    void AllocateRegister(Operand source, MemoryOperand memOp, int srcIndex)
+                    Operand AllocateRegister(Operand local)
                     {
-                        LocalInfo info = locInfo[source.GetLocalNumber() - 1];
+                        ref LocalInfo info = ref GetLocalInfo(local);
 
-                        info.UseCount++;
+                        info.UsesAllocated++;
 
-                        Debug.Assert(info.UseCount <= info.Uses);
+                        Debug.Assert(info.UsesAllocated <= info.Uses);
 
                         if (info.Register != -1)
                         {
-                            Operand reg = Register(info.Register, source.Type.ToRegisterType(), source.Type);
+                            Operand reg = Register(info.Register, local.Type.ToRegisterType(), local.Type);
 
-                            if (memOp != null)
+                            if (info.UsesAllocated == info.Uses)
                             {
-                                if (srcIndex == 0)
+                                if (local.Type.IsInteger())
-                                {
-                                    memOp.BaseAddress = reg;
-                                }
-                                else /* if (srcIndex == 1) */
-                                {
-                                    memOp.Index = reg;
-                                }
-                            }
-                            else
-                            {
-                                node.SetSource(srcIndex, reg);
-                            }
-
-                            if (info.UseCount == info.Uses && !info.PreAllocated)
-                            {
-                                if (source.Type.IsInteger())
                                 {
                                     intLocalFreeRegisters |= 1 << info.Register;
                                 }
@@ -236,8 +242,44 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                                     vecLocalFreeRegisters |= 1 << info.Register;
                                 }
                             }
+
+                            return reg;
                         }
-                        else if (node is Operation operation && operation.Instruction == Instruction.Copy)
+                        else
+                        {
+                            Operand temp = info.Temp;
+
+                            if (temp == default || info.Sequence != sequence)
+                            {
+                                temp = local.Type.IsInteger()
+                                    ? GetSpillTemp(local, intSpillTempRegisters, ref intLocalUse)
+                                    : GetSpillTemp(local, vecSpillTempRegisters, ref vecLocalUse);
+
+                                info.Sequence = sequence;
+                                info.Temp = temp;
+                            }
+
+                            Operation fillOp = Operation(Instruction.Fill, temp, Const(info.SpillOffset));
+
+                            block.Operations.AddBefore(node, fillOp);
+
+                            return temp;
+                        }
+                    }
+
+                    bool folded = false;
+
+                    // If operation is a copy of a local and that local is living on the stack, we turn the copy into
+                    // a fill, instead of inserting a fill before it.
+                    if (node.Instruction == Instruction.Copy)
+                    {
+                        Operand source = node.GetSource(0);
+
+                        if (source.Kind == OperandKind.LocalVariable)
+                        {
+                            ref LocalInfo info = ref GetLocalInfo(source);
+
+                            if (info.Register == -1)
                             {
                                 Operation fillOp = Operation(Instruction.Fill, node.Destination, Const(info.SpillOffset));
 
@@ -245,63 +287,35 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                                 block.Operations.Remove(node);
 
                                 node = fillOp;
-                        }
-                        else
-                        {
-                            Operand temp = info.Temp;
 
-                            if (temp == null || info.Sequence != sequence)
+                                folded = true;
-                            {
-                                temp = source.Type.IsInteger()
-                                    ? GetSpillTemp(source, intSpillTempRegisters, ref intLocalUse)
-                                    : GetSpillTemp(source, vecSpillTempRegisters, ref vecLocalUse);
-
-                                info.Sequence = sequence;
-                                info.Temp = temp;
                             }
-
-                            if (memOp != null)
-                            {
-                                if (srcIndex == 0)
-                                {
-                                    memOp.BaseAddress = temp;
-                                }
-                                else /* if (srcIndex == 1) */
-                                {
-                                    memOp.Index = temp;
-                                }
-                            }
-                            else
-                            {
-                                node.SetSource(srcIndex, temp);
-                            }
-
-                            Operation fillOp = Operation(Instruction.Fill, temp, Const(info.SpillOffset));
-
-                            block.Operations.AddBefore(node, fillOp);
                         }
                     }
 
-                    for (int srcIndex = 0; srcIndex < node.SourcesCount; srcIndex++)
+                    if (!folded)
                     {
-                        Operand source = node.GetSource(srcIndex);
+                        for (int i = 0; i < node.SourcesCount; i++)
+                        {
+                            Operand source = node.GetSource(i);
 
                             if (source.Kind == OperandKind.LocalVariable)
                             {
-                            AllocateRegister(source, null, srcIndex);
+                                node.SetSource(i, AllocateRegister(source));
                             }
                             else if (source.Kind == OperandKind.Memory)
                             {
-                            MemoryOperand memOp = (MemoryOperand)source;
+                                MemoryOperand memOp = source.GetMemory();
 
-                            if (memOp.BaseAddress != null)
+                                if (memOp.BaseAddress != default)
                                 {
-                                AllocateRegister(memOp.BaseAddress, memOp, 0);
+                                    memOp.BaseAddress = AllocateRegister(memOp.BaseAddress);
                                 }
 
-                            if (memOp.Index != null)
+                                if (memOp.Index != default)
                                 {
-                                AllocateRegister(memOp.Index, memOp, 1);
+                                    memOp.Index = AllocateRegister(memOp.Index);
+                                }
                             }
                         }
                     }
@@ -309,18 +323,18 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                     int intLocalAsg = 0;
                     int vecLocalAsg = 0;
 
-                    for (int dstIndex = 0; dstIndex < node.DestinationsCount; dstIndex++)
+                    for (int i = 0; i < node.DestinationsCount; i++)
                     {
-                        Operand dest = node.GetDestination(dstIndex);
+                        Operand dest = node.GetDestination(i);
 
                         if (dest.Kind != OperandKind.LocalVariable)
                         {
                             continue;
                         }
 
-                        LocalInfo info = locInfo[dest.GetLocalNumber() - 1];
+                        ref LocalInfo info = ref GetLocalInfo(dest);
 
-                        if (info.UseCount == 0 && !info.PreAllocated)
+                        if (info.UsesAllocated == 0)
                         {
                             int mask = dest.Type.IsInteger()
                                 ? intLocalFreeRegisters
@@ -350,19 +364,19 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                             }
                         }
 
-                        info.UseCount++;
+                        info.UsesAllocated++;
 
-                        Debug.Assert(info.UseCount <= info.Uses);
+                        Debug.Assert(info.UsesAllocated <= info.Uses);
 
                         if (info.Register != -1)
                         {
-                            node.SetDestination(dstIndex, Register(info.Register, dest.Type.ToRegisterType(), dest.Type));
+                            node.SetDestination(i, Register(info.Register, dest.Type.ToRegisterType(), dest.Type));
                         }
                         else
                         {
                             Operand temp = info.Temp;
 
-                            if (temp == null || info.Sequence != sequence)
+                            if (temp == default || info.Sequence != sequence)
                             {
                                 temp = dest.Type.IsInteger()
                                     ? GetSpillTemp(dest, intSpillTempRegisters, ref intLocalAsg)
@@ -372,9 +386,9 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                                 info.Temp     = temp;
                             }
 
-                            node.SetDestination(dstIndex, temp);
+                            node.SetDestination(i, temp);
 
-                            Operation spillOp = Operation(Instruction.Spill, null, Const(info.SpillOffset), temp);
+                            Operation spillOp = Operation(Instruction.Spill, default, Const(info.SpillOffset), temp);
 
                             block.Operations.AddAfter(node, spillOp);
 
@@ -435,7 +449,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
 
         private static int UsesCount(Operand local)
         {
-            return local.Assignments.Count + local.Uses.Count;
+            return local.AssignmentsCount + local.UsesCount;
         }
     }
 }

ARMeilleure/CodeGen/RegisterAllocators/LinearScanAllocator.cs

@@ -29,11 +29,11 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
 
         private LiveInterval[] _parentIntervals;
 
-        private List<(IntrusiveList<Node>, Node)> _operationNodes;
+        private List<(IntrusiveList<Operation>, Operation)> _operationNodes;
 
         private int _operationsCount;
 
-        private class AllocationContext : IDisposable
+        private class AllocationContext
         {
             public RegisterMasks Masks { get; }
 
@@ -50,10 +50,8 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                 StackAlloc = stackAlloc;
                 Masks      = masks;
 
-                BitMapPool.PrepareBitMapPool();
+                Active   = new BitMap(Allocators.Default, intervalsCount);
-
+                Inactive = new BitMap(Allocators.Default, intervalsCount);
-                Active   = BitMapPool.Allocate(intervalsCount);
-                Inactive = BitMapPool.Allocate(intervalsCount);
             }
 
             public void MoveActiveToInactive(int bit)
@@ -72,11 +70,6 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
 
                 dest.Set(bit);
             }
-
-            public void Dispose()
-            {
-                BitMapPool.ResetBitMapPool();
-            }
         }
 
         public AllocationResult RunPass(
@@ -86,7 +79,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
         {
             NumberLocals(cfg);
 
-            using AllocationContext context = new AllocationContext(stackAlloc, regMasks, _intervals.Count);
+            var context = new AllocationContext(stackAlloc, regMasks, _intervals.Count);
 
             BuildIntervals(cfg, context);
 
@@ -588,7 +581,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
 
                 int splitPosition = kv.Key;
 
-                (IntrusiveList<Node> nodes, Node node) = GetOperationNode(splitPosition);
+                (IntrusiveList<Operation> nodes, Operation node) = GetOperationNode(splitPosition);
 
                 Operation[] sequence = copyResolver.Sequence();
 
@@ -621,9 +614,9 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                     continue;
                 }
 
-                bool hasSingleOrNoSuccessor = block.SuccessorCount <= 1;
+                bool hasSingleOrNoSuccessor = block.SuccessorsCount <= 1;
 
-                for (int i = 0; i < block.SuccessorCount; i++)
+                for (int i = 0; i < block.SuccessorsCount; i++)
                 {
                     BasicBlock successor = block.GetSuccessor(i);
 
@@ -677,7 +670,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                     {
                         successor.Operations.AddFirst(sequence[0]);
 
-                        Node prependNode = sequence[0];
+                        Operation prependNode = sequence[0];
 
                         for (int index = 1; index < sequence.Length; index++)
                         {
@@ -710,7 +703,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             for (int i = usePositions.Count - 1; i >= 0; i--)
             {
                 int usePosition = -usePositions[i];
-                (_, Node operation) = GetOperationNode(usePosition);
+                (_, Operation operation) = GetOperationNode(usePosition);
 
                 for (int index = 0; index < operation.SourcesCount; index++)
                 {
@@ -722,7 +715,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                     }
                     else if (source.Kind == OperandKind.Memory)
                     {
-                        MemoryOperand memOp = (MemoryOperand)source;
+                        MemoryOperand memOp = source.GetMemory();
 
                         if (memOp.BaseAddress == current.Local)
                         {
@@ -752,20 +745,20 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
         {
             Debug.Assert(!interval.IsSpilled, "Spilled intervals are not allowed.");
 
-            return OperandHelper.Register(
+            return Operand.Factory.Register(
                 interval.Register.Index,
                 interval.Register.Type,
                 interval.Local.Type);
         }
 
-        private (IntrusiveList<Node>, Node) GetOperationNode(int position)
+        private (IntrusiveList<Operation>, Operation) GetOperationNode(int position)
         {
             return _operationNodes[position / InstructionGap];
         }
 
         private void NumberLocals(ControlFlowGraph cfg)
         {
-            _operationNodes = new List<(IntrusiveList<Node>, Node)>();
+            _operationNodes = new List<(IntrusiveList<Operation>, Operation)>();
 
             _intervals = new List<LiveInterval>();
 
@@ -783,13 +776,14 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             {
                 BasicBlock block = cfg.PostOrderBlocks[index];
 
-                for (Node node = block.Operations.First; node != null; node = node.ListNext)
+                for (Operation node = block.Operations.First; node != default; node = node.ListNext)
                 {
                     _operationNodes.Add((block.Operations, node));
 
                     for (int i = 0; i < node.DestinationsCount; i++)
                     {
                         Operand dest = node.GetDestination(i);
+
                         if (dest.Kind == OperandKind.LocalVariable && visited.Add(dest))
                         {
                             dest.NumberLocal(_intervals.Count);
@@ -804,7 +798,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                 if (block.Operations.Count == 0)
                 {
                     // Pretend we have a dummy instruction on the empty block.
-                    _operationNodes.Add((null, null));
+                    _operationNodes.Add((default, default));
 
                     _operationsCount += InstructionGap;
                 }
@@ -825,10 +819,10 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             // Compute local live sets.
             for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext)
             {
-                BitMap liveGen  = BitMapPool.Allocate(mapSize);
+                BitMap liveGen  = new BitMap(Allocators.Default, mapSize);
-                BitMap liveKill = BitMapPool.Allocate(mapSize);
+                BitMap liveKill = new BitMap(Allocators.Default, mapSize);
 
-                for (Node node = block.Operations.First; node != null; node = node.ListNext)
+                for (Operation node = block.Operations.First; node != default; node = node.ListNext)
                 {
                     Sources(node, (source) =>
                     {
@@ -857,8 +851,8 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
 
             for (int index = 0; index < cfg.Blocks.Count; index++)
             {
-                blkLiveIn [index] = BitMapPool.Allocate(mapSize);
+                blkLiveIn [index] = new BitMap(Allocators.Default, mapSize);
-                blkLiveOut[index] = BitMapPool.Allocate(mapSize);
+                blkLiveOut[index] = new BitMap(Allocators.Default, mapSize);
             }
 
             bool modified;
@@ -873,7 +867,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
 
                     BitMap liveOut = blkLiveOut[block.Index];
 
-                    for (int i = 0; i < block.SuccessorCount; i++)
+                    for (int i = 0; i < block.SuccessorsCount; i++)
                     {
                         BasicBlock succ = block.GetSuccessor(i);
 
@@ -926,7 +920,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                     continue;
                 }
 
-                foreach (Node node in BottomOperations(block))
+                foreach (Operation node in BottomOperations(block))
                 {
                     operationPos -= InstructionGap;
 
@@ -947,7 +941,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                         interval.AddUsePosition(operationPos);
                     });
 
-                    if (node is Operation operation && operation.Instruction == Instruction.Call)
+                    if (node.Instruction == Instruction.Call)
                     {
                         AddIntervalCallerSavedReg(context.Masks.IntCallerSavedRegisters, operationPos, RegisterType.Integer);
                         AddIntervalCallerSavedReg(context.Masks.VecCallerSavedRegisters, operationPos, RegisterType.Vector);
@@ -993,11 +987,11 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             return (register.Index << 1) | (register.Type == RegisterType.Vector ? 1 : 0);
         }
 
-        private static IEnumerable<Node> BottomOperations(BasicBlock block)
+        private static IEnumerable<Operation> BottomOperations(BasicBlock block)
         {
-            Node node = block.Operations.Last;
+            Operation node = block.Operations.Last;
 
-            while (node != null && !(node is PhiNode))
+            while (node != default)
             {
                 yield return node;
 
@@ -1005,7 +999,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             }
         }
 
-        private static void Sources(Node node, Action<Operand> action)
+        private static void Sources(Operation node, Action<Operand> action)
         {
             for (int index = 0; index < node.SourcesCount; index++)
             {
@@ -1017,14 +1011,14 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                 }
                 else if (source.Kind == OperandKind.Memory)
                 {
-                    MemoryOperand memOp = (MemoryOperand)source;
+                    MemoryOperand memOp = source.GetMemory();
 
-                    if (memOp.BaseAddress != null)
+                    if (memOp.BaseAddress != default)
                     {
                         action(memOp.BaseAddress);
                     }
 
-                    if (memOp.Index != null)
+                    if (memOp.Index != default)
                     {
                         action(memOp.Index);
                     }

ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs

@@ -34,7 +34,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
 
         public bool IsEmpty => _ranges.Count == 0;
 
-        public LiveInterval(Operand local = null, LiveInterval parent = null)
+        public LiveInterval(Operand local = default, LiveInterval parent = null)
         {
             Local   = local;
             _parent = parent ?? this;

ARMeilleure/CodeGen/X86/Assembler.cs

@@ -329,12 +329,12 @@ namespace ARMeilleure.CodeGen.X86
 
         public void Bswap(Operand dest)
         {
-            WriteInstruction(dest, null, dest.Type, X86Instruction.Bswap);
+            WriteInstruction(dest, default, dest.Type, X86Instruction.Bswap);
         }
 
         public void Call(Operand dest)
         {
-            WriteInstruction(dest, null, OperandType.None, X86Instruction.Call);
+            WriteInstruction(dest, default, OperandType.None, X86Instruction.Call);
         }
 
         public void Cdq()
@@ -346,7 +346,7 @@ namespace ARMeilleure.CodeGen.X86
         {
             InstructionInfo info = _instTable[(int)X86Instruction.Cmovcc];
 
-            WriteOpCode(dest, null, source, type, info.Flags, info.OpRRM | (int)condition, rrm: true);
+            WriteOpCode(dest, default, source, type, info.Flags, info.OpRRM | (int)condition, rrm: true);
         }
 
         public void Cmp(Operand src1, Operand src2, OperandType type)
@@ -360,30 +360,38 @@ namespace ARMeilleure.CodeGen.X86
             WriteByte(0x99);
         }
 
-        public void Cmpxchg(MemoryOperand memOp, Operand src)
+        public void Cmpxchg(Operand memOp, Operand src)
         {
+            Debug.Assert(memOp.Kind == OperandKind.Memory);
+
             WriteByte(LockPrefix);
 
             WriteInstruction(memOp, src, src.Type, X86Instruction.Cmpxchg);
         }
 
-        public void Cmpxchg16(MemoryOperand memOp, Operand src)
+        public void Cmpxchg16(Operand memOp, Operand src)
         {
+            Debug.Assert(memOp.Kind == OperandKind.Memory);
+
             WriteByte(LockPrefix);
             WriteByte(0x66);
 
             WriteInstruction(memOp, src, src.Type, X86Instruction.Cmpxchg);
         }
 
-        public void Cmpxchg16b(MemoryOperand memOp)
+        public void Cmpxchg16b(Operand memOp)
         {
+            Debug.Assert(memOp.Kind == OperandKind.Memory);
+
             WriteByte(LockPrefix);
 
-            WriteInstruction(memOp, null, OperandType.None, X86Instruction.Cmpxchg16b);
+            WriteInstruction(memOp, default, OperandType.None, X86Instruction.Cmpxchg16b);
         }
 
-        public void Cmpxchg8(MemoryOperand memOp, Operand src)
+        public void Cmpxchg8(Operand memOp, Operand src)
         {
+            Debug.Assert(memOp.Kind == OperandKind.Memory);
+
             WriteByte(LockPrefix);
 
             WriteInstruction(memOp, src, src.Type, X86Instruction.Cmpxchg8);
@@ -391,12 +399,12 @@ namespace ARMeilleure.CodeGen.X86
 
         public void Comisd(Operand src1, Operand src2)
         {
-            WriteInstruction(src1, null, src2, X86Instruction.Comisd);
+            WriteInstruction(src1, default, src2, X86Instruction.Comisd);
         }
 
         public void Comiss(Operand src1, Operand src2)
         {
-            WriteInstruction(src1, null, src2, X86Instruction.Comiss);
+            WriteInstruction(src1, default, src2, X86Instruction.Comiss);
         }
 
         public void Cvtsd2ss(Operand dest, Operand src1, Operand src2)
@@ -421,7 +429,7 @@ namespace ARMeilleure.CodeGen.X86
 
         public void Div(Operand source)
         {
-            WriteInstruction(null, source, source.Type, X86Instruction.Div);
+            WriteInstruction(default, source, source.Type, X86Instruction.Div);
         }
 
         public void Divsd(Operand dest, Operand src1, Operand src2)
@@ -436,12 +444,12 @@ namespace ARMeilleure.CodeGen.X86
 
         public void Idiv(Operand source)
         {
-            WriteInstruction(null, source, source.Type, X86Instruction.Idiv);
+            WriteInstruction(default, source, source.Type, X86Instruction.Idiv);
         }
 
         public void Imul(Operand source)
         {
-            WriteInstruction(null, source, source.Type, X86Instruction.Imul128);
+            WriteInstruction(default, source, source.Type, X86Instruction.Imul128);
         }
 
         public void Imul(Operand dest, Operand source, OperandType type)
@@ -465,13 +473,13 @@ namespace ARMeilleure.CodeGen.X86
 
             if (IsImm8(src2.Value, src2.Type) && info.OpRMImm8 != BadOp)
             {
-                WriteOpCode(dest, null, src1, type, info.Flags, info.OpRMImm8, rrm: true);
+                WriteOpCode(dest, default, src1, type, info.Flags, info.OpRMImm8, rrm: true);
 
                 WriteByte(src2.AsByte());
             }
             else if (IsImm32(src2.Value, src2.Type) && info.OpRMImm32 != BadOp)
             {
-                WriteOpCode(dest, null, src1, type, info.Flags, info.OpRMImm32, rrm: true);
+                WriteOpCode(dest, default, src1, type, info.Flags, info.OpRMImm32, rrm: true);
 
                 WriteInt32(src2.AsInt32());
             }
@@ -531,12 +539,12 @@ namespace ARMeilleure.CodeGen.X86
 
         public void Jmp(Operand dest)
         {
-            WriteInstruction(dest, null, OperandType.None, X86Instruction.Jmp);
+            WriteInstruction(dest, default, OperandType.None, X86Instruction.Jmp);
         }
 
         public void Ldmxcsr(Operand dest)
         {
-            WriteInstruction(dest, null, OperandType.I32, X86Instruction.Ldmxcsr);
+            WriteInstruction(dest, default, OperandType.I32, X86Instruction.Ldmxcsr);
         }
 
         public void Lea(Operand dest, Operand source, OperandType type)
@@ -565,17 +573,17 @@ namespace ARMeilleure.CodeGen.X86
 
             if (source.Type.IsInteger() || source.Kind == OperandKind.Memory)
             {
-                WriteOpCode(dest, null, source, OperandType.None, info.Flags, info.OpRRM, rrm: true);
+                WriteOpCode(dest, default, source, OperandType.None, info.Flags, info.OpRRM, rrm: true);
             }
             else
             {
-                WriteOpCode(dest, null, source, OperandType.None, info.Flags, info.OpRMR);
+                WriteOpCode(dest, default, source, OperandType.None, info.Flags, info.OpRMR);
             }
         }
 
         public void Movdqu(Operand dest, Operand source)
         {
-            WriteInstruction(dest, null, source, X86Instruction.Movdqu);
+            WriteInstruction(dest, default, source, X86Instruction.Movdqu);
         }
 
         public void Movhlps(Operand dest, Operand src1, Operand src2)
@@ -596,11 +604,11 @@ namespace ARMeilleure.CodeGen.X86
 
             if (source.Type.IsInteger() || source.Kind == OperandKind.Memory)
             {
-                WriteOpCode(dest, null, source, OperandType.None, flags, info.OpRRM, rrm: true);
+                WriteOpCode(dest, default, source, OperandType.None, flags, info.OpRRM, rrm: true);
             }
             else if (dest.Type.IsInteger() || dest.Kind == OperandKind.Memory)
             {
-                WriteOpCode(dest, null, source, OperandType.None, flags, info.OpRMR);
+                WriteOpCode(dest, default, source, OperandType.None, flags, info.OpRMR);
             }
             else
             {
@@ -645,7 +653,7 @@ namespace ARMeilleure.CodeGen.X86
 
         public void Mul(Operand source)
         {
-            WriteInstruction(null, source, source.Type, X86Instruction.Mul128);
+            WriteInstruction(default, source, source.Type, X86Instruction.Mul128);
         }
 
         public void Mulsd(Operand dest, Operand src1, Operand src2)
@@ -660,12 +668,12 @@ namespace ARMeilleure.CodeGen.X86
 
         public void Neg(Operand dest)
         {
-            WriteInstruction(dest, null, dest.Type, X86Instruction.Neg);
+            WriteInstruction(dest, default, dest.Type, X86Instruction.Neg);
         }
 
         public void Not(Operand dest)
         {
-            WriteInstruction(dest, null, dest.Type, X86Instruction.Not);
+            WriteInstruction(dest, default, dest.Type, X86Instruction.Not);
         }
 
         public void Or(Operand dest, Operand source, OperandType type)
@@ -675,7 +683,7 @@ namespace ARMeilleure.CodeGen.X86
 
         public void Pclmulqdq(Operand dest, Operand source, byte imm)
         {
-            WriteInstruction(dest, null, source, X86Instruction.Pclmulqdq);
+            WriteInstruction(dest, default, source, X86Instruction.Pclmulqdq);
 
             WriteByte(imm);
         }
@@ -687,28 +695,28 @@ namespace ARMeilleure.CodeGen.X86
 
         public void Pextrb(Operand dest, Operand source, byte imm)
         {
-            WriteInstruction(dest, null, source, X86Instruction.Pextrb);
+            WriteInstruction(dest, default, source, X86Instruction.Pextrb);
 
             WriteByte(imm);
         }
 
         public void Pextrd(Operand dest, Operand source, byte imm)
         {
-            WriteInstruction(dest, null, source, X86Instruction.Pextrd);
+            WriteInstruction(dest, default, source, X86Instruction.Pextrd);
 
             WriteByte(imm);
         }
 
         public void Pextrq(Operand dest, Operand source, byte imm)
         {
-            WriteInstruction(dest, null, source, X86Instruction.Pextrq);
+            WriteInstruction(dest, default, source, X86Instruction.Pextrq);
 
             WriteByte(imm);
         }
 
         public void Pextrw(Operand dest, Operand source, byte imm)
         {
-            WriteInstruction(dest, null, source, X86Instruction.Pextrw);
+            WriteInstruction(dest, default, source, X86Instruction.Pextrw);
 
             WriteByte(imm);
         }
@@ -749,7 +757,7 @@ namespace ARMeilleure.CodeGen.X86
             }
             else
             {
-                WriteInstruction(dest, null, dest.Type, X86Instruction.Pop);
+                WriteInstruction(dest, default, dest.Type, X86Instruction.Pop);
             }
         }
 
@@ -760,7 +768,7 @@ namespace ARMeilleure.CodeGen.X86
 
         public void Pshufd(Operand dest, Operand source, byte imm)
         {
-            WriteInstruction(dest, null, source, X86Instruction.Pshufd);
+            WriteInstruction(dest, default, source, X86Instruction.Pshufd);
 
             WriteByte(imm);
         }
@@ -773,7 +781,7 @@ namespace ARMeilleure.CodeGen.X86
             }
             else
             {
-                WriteInstruction(null, source, source.Type, X86Instruction.Push);
+                WriteInstruction(default, source, source.Type, X86Instruction.Push);
             }
         }
 
@@ -806,12 +814,12 @@ namespace ARMeilleure.CodeGen.X86
         {
             InstructionInfo info = _instTable[(int)X86Instruction.Setcc];
 
-            WriteOpCode(dest, null, null, OperandType.None, info.Flags, info.OpRRM | (int)condition);
+            WriteOpCode(dest, default, default, OperandType.None, info.Flags, info.OpRRM | (int)condition);
         }
 
         public void Stmxcsr(Operand dest)
         {
-            WriteInstruction(dest, null, OperandType.I32, X86Instruction.Stmxcsr);
+            WriteInstruction(dest, default, OperandType.I32, X86Instruction.Stmxcsr);
         }
 
         public void Sub(Operand dest, Operand source, OperandType type)
@@ -850,7 +858,7 @@ namespace ARMeilleure.CodeGen.X86
             Operand source,
             OperandType type = OperandType.None)
         {
-            WriteInstruction(dest, null, source, inst, type);
+            WriteInstruction(dest, default, source, inst, type);
         }
 
         public void WriteInstruction(X86Instruction inst, Operand dest, Operand src1, Operand src2)
@@ -877,7 +885,7 @@ namespace ARMeilleure.CodeGen.X86
 
         public void WriteInstruction(X86Instruction inst, Operand dest, Operand source, byte imm)
         {
-            WriteInstruction(dest, null, source, inst);
+            WriteInstruction(dest, default, source, inst);
 
             WriteByte(imm);
         }
@@ -917,11 +925,11 @@ namespace ARMeilleure.CodeGen.X86
 
                 Debug.Assert(shiftReg == X86Register.Rcx, $"Invalid shift register \"{shiftReg}\".");
 
-                source = null;
+                source = default;
             }
             else if (source.Kind == OperandKind.Constant)
             {
-                source = source.With((int)source.Value & (dest.Type == OperandType.I32 ? 0x1f : 0x3f));
+                source = Operand.Factory.Const((int)source.Value & (dest.Type == OperandType.I32 ? 0x1f : 0x3f));
             }
 
             WriteInstruction(dest, source, type, inst);
@@ -931,7 +939,7 @@ namespace ARMeilleure.CodeGen.X86
         {
             InstructionInfo info = _instTable[(int)inst];
 
-            if (source != null)
+            if (source != default)
             {
                 if (source.Kind == OperandKind.Constant)
                 {
@@ -939,29 +947,29 @@ namespace ARMeilleure.CodeGen.X86
 
                     if (inst == X86Instruction.Mov8)
                     {
-                        WriteOpCode(dest, null, null, type, info.Flags, info.OpRMImm8);
+                        WriteOpCode(dest, default, default, type, info.Flags, info.OpRMImm8);
 
                         WriteByte((byte)imm);
                     }
                     else if (inst == X86Instruction.Mov16)
                     {
-                        WriteOpCode(dest, null, null, type, info.Flags, info.OpRMImm32);
+                        WriteOpCode(dest, default, default, type, info.Flags, info.OpRMImm32);
 
                         WriteInt16((short)imm);
                     }
                     else if (IsImm8(imm, type) && info.OpRMImm8 != BadOp)
                     {
-                        WriteOpCode(dest, null, null, type, info.Flags, info.OpRMImm8);
+                        WriteOpCode(dest, default, default, type, info.Flags, info.OpRMImm8);
 
                         WriteByte((byte)imm);
                     }
                     else if (!source.Relocatable && IsImm32(imm, type) && info.OpRMImm32 != BadOp)
                     {
-                        WriteOpCode(dest, null, null, type, info.Flags, info.OpRMImm32);
+                        WriteOpCode(dest, default, default, type, info.Flags, info.OpRMImm32);
 
                         WriteInt32((int)imm);
                     }
-                    else if (dest?.Kind == OperandKind.Register && info.OpRImm64 != BadOp)
+                    else if (dest != default && dest.Kind == OperandKind.Register && info.OpRImm64 != BadOp)
                     {
                         int rexPrefix = GetRexPrefix(dest, source, type, rrm: false);
 
@@ -972,7 +980,7 @@ namespace ARMeilleure.CodeGen.X86
 
                         WriteByte((byte)(info.OpRImm64 + (dest.GetRegister().Index & 0b111)));
 
-                        if (_ptcInfo != null && source.Relocatable)
+                        if (_ptcInfo != default && source.Relocatable)
                         {
                             _ptcInfo.WriteRelocEntry(new RelocEntry((int)_stream.Position, source.Symbol));
                         }
@@ -986,11 +994,11 @@ namespace ARMeilleure.CodeGen.X86
                 }
                 else if (source.Kind == OperandKind.Register && info.OpRMR != BadOp)
                 {
-                    WriteOpCode(dest, null, source, type, info.Flags, info.OpRMR);
+                    WriteOpCode(dest, default, source, type, info.Flags, info.OpRMR);
                 }
                 else if (info.OpRRM != BadOp)
                 {
-                    WriteOpCode(dest, null, source, type, info.Flags, info.OpRRM, rrm: true);
+                    WriteOpCode(dest, default, source, type, info.Flags, info.OpRRM, rrm: true);
                 }
                 else
                 {
@@ -999,11 +1007,11 @@ namespace ARMeilleure.CodeGen.X86
             }
             else if (info.OpRRM != BadOp)
             {
-                WriteOpCode(dest, null, source, type, info.Flags, info.OpRRM, rrm: true);
+                WriteOpCode(dest, default, source, type, info.Flags, info.OpRRM, rrm: true);
             }
             else if (info.OpRMR != BadOp)
             {
-                WriteOpCode(dest, null, source, type, info.Flags, info.OpRMR);
+                WriteOpCode(dest, default, source, type, info.Flags, info.OpRMR);
             }
             else
             {
@@ -1020,7 +1028,7 @@ namespace ARMeilleure.CodeGen.X86
         {
             InstructionInfo info = _instTable[(int)inst];
 
-            if (src2 != null)
+            if (src2 != default)
             {
                 if (src2.Kind == OperandKind.Constant)
                 {
@@ -1028,7 +1036,7 @@ namespace ARMeilleure.CodeGen.X86
 
                     if ((byte)imm == imm && info.OpRMImm8 != BadOp)
                     {
-                        WriteOpCode(dest, src1, null, type, info.Flags, info.OpRMImm8);
+                        WriteOpCode(dest, src1, default, type, info.Flags, info.OpRMImm8);
 
                         WriteByte((byte)imm);
                     }
@@ -1082,9 +1090,10 @@ namespace ARMeilleure.CodeGen.X86
 
             int modRM = (opCode >> OpModRMBits) << 3;
 
-            MemoryOperand memOp = null;
+            MemoryOperand memOp = default;
+            bool hasMemOp = false;
 
-            if (dest != null)
+            if (dest != default)
             {
                 if (dest.Kind == OperandKind.Register)
                 {
@@ -1099,7 +1108,8 @@ namespace ARMeilleure.CodeGen.X86
                 }
                 else if (dest.Kind == OperandKind.Memory)
                 {
-                    memOp = dest as MemoryOperand;
+                    memOp = dest.GetMemory();
+                    hasMemOp = true;
                 }
                 else
                 {
@@ -1107,7 +1117,7 @@ namespace ARMeilleure.CodeGen.X86
                 }
             }
 
-            if (src2 != null)
+            if (src2 != default)
             {
                 if (src2.Kind == OperandKind.Register)
                 {
@@ -1120,9 +1130,10 @@ namespace ARMeilleure.CodeGen.X86
                         rexPrefix |= RexPrefix;
                     }
                 }
-                else if (src2.Kind == OperandKind.Memory && memOp == null)
+                else if (src2.Kind == OperandKind.Memory && !hasMemOp)
                 {
-                    memOp = src2 as MemoryOperand;
+                    memOp = src2.GetMemory();
+                    hasMemOp = true;
                 }
                 else
                 {
@@ -1135,14 +1146,14 @@ namespace ARMeilleure.CodeGen.X86
 
             int sib = 0;
 
-            if (memOp != null)
+            if (hasMemOp)
             {
                 // Either source or destination is a memory operand.
                 Register baseReg = memOp.BaseAddress.GetRegister();
 
                 X86Register baseRegLow = (X86Register)(baseReg.Index & 0b111);
 
-                needsSibByte      = memOp.Index != null     || baseRegLow == X86Register.Rsp;
+                needsSibByte      = memOp.Index != default  || baseRegLow == X86Register.Rsp;
                 needsDisplacement = memOp.Displacement != 0 || baseRegLow == X86Register.Rbp;
 
                 if (needsDisplacement)
@@ -1168,7 +1179,7 @@ namespace ARMeilleure.CodeGen.X86
                 {
                     sib = (int)baseRegLow;
 
-                    if (memOp.Index != null)
+                    if (memOp.Index != default)
                     {
                         int indexReg = memOp.Index.GetRegister().Index;
 
@@ -1217,7 +1228,7 @@ namespace ARMeilleure.CodeGen.X86
                     _ => 0
                 };
 
-                if (src1 != null)
+                if (src1 != default)
                 {
                     vexByte2 |= (src1.GetRegister().Index ^ 0xf) << 3;
                 }
@@ -1284,7 +1295,7 @@ namespace ARMeilleure.CodeGen.X86
                 }
             }
 
-            if (dest != null && (flags & InstructionFlags.RegOnly) != 0)
+            if (dest != default && (flags & InstructionFlags.RegOnly) != 0)
             {
                 opCode += dest.GetRegister().Index & 7;
             }
@@ -1353,12 +1364,12 @@ namespace ARMeilleure.CodeGen.X86
                 }
             }
 
-            if (dest != null && dest.Kind == OperandKind.Register)
+            if (dest != default && dest.Kind == OperandKind.Register)
             {
                 SetRegisterHighBit(dest.GetRegister(), rrm ? 2 : 0);
             }
 
-            if (source != null && source.Kind == OperandKind.Register)
+            if (source != default && source.Kind == OperandKind.Register)
             {
                 SetRegisterHighBit(source.GetRegister(), rrm ? 0 : 2);
             }

ARMeilleure/CodeGen/X86/CodeGenerator.cs

@@ -11,8 +11,7 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Numerics;
-
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
 
 namespace ARMeilleure.CodeGen.X86
 {
@@ -162,20 +161,18 @@ namespace ARMeilleure.CodeGen.X86
                 {
                     context.EnterBlock(block);
 
-                    for (Node node = block.Operations.First; node != null; node = node.ListNext)
+                    for (Operation node = block.Operations.First; node != default; node = node.ListNext)
                     {
-                        if (node is Operation operation)
+                        GenerateOperation(context, node);
-                        {
-                            GenerateOperation(context, operation);
-                        }
                     }
 
-                    if (block.SuccessorCount == 0)
+                    if (block.SuccessorsCount == 0)
                     {
                         // The only blocks which can have 0 successors are exit blocks.
-                        Debug.Assert(block.Operations.Last is Operation operation &&
+                        Operation last = block.Operations.Last;
-                                     (operation.Instruction == Instruction.Tailcall ||
+
-                                      operation.Instruction == Instruction.Return));
+                        Debug.Assert(last.Instruction == Instruction.Tailcall ||
+                                     last.Instruction == Instruction.Return);
                     }
                     else
                     {
@@ -205,9 +202,7 @@ namespace ARMeilleure.CodeGen.X86
         {
             if (operation.Instruction == Instruction.Extended)
             {
-                IntrinsicOperation intrinOp = (IntrinsicOperation)operation;
+                IntrinsicInfo info = IntrinsicTable.GetInfo(operation.Intrinsic);
-
-                IntrinsicInfo info = IntrinsicTable.GetInfo(intrinOp.Intrinsic);
 
                 switch (info.Type)
                 {
@@ -217,7 +212,7 @@ namespace ARMeilleure.CodeGen.X86
                         Operand src1 = operation.GetSource(0);
                         Operand src2 = operation.GetSource(1);
 
-                        switch (intrinOp.Intrinsic)
+                        switch (operation.Intrinsic)
                         {
                             case Intrinsic.X86Comisdeq:
                                 context.Assembler.Comisd(src1, src2);
@@ -266,14 +261,13 @@ namespace ARMeilleure.CodeGen.X86
                         int offs = offset.AsInt32() + context.CallArgsRegionSize;
 
                         Operand rsp = Register(X86Register.Rsp);
-
+                        Operand memOp = MemoryOp(OperandType.I32, rsp, default, Multiplier.x1, offs);
-                        MemoryOperand memOp = MemoryOp(OperandType.I32, rsp, null, Multiplier.x1, offs);
 
                         Debug.Assert(HardwareCapabilities.SupportsSse || HardwareCapabilities.SupportsVexEncoding);
 
                         context.Assembler.Stmxcsr(memOp);
 
-                        if (intrinOp.Intrinsic == Intrinsic.X86Mxcsrmb)
+                        if (operation.Intrinsic == Intrinsic.X86Mxcsrmb)
                         {
                             context.Assembler.Or(memOp, bits, OperandType.I32);
                         }
@@ -324,7 +318,7 @@ namespace ARMeilleure.CodeGen.X86
 
                         Debug.Assert(dest.Type.IsInteger() && !source.Type.IsInteger());
 
-                        if (intrinOp.Intrinsic == Intrinsic.X86Cvtsi2si)
+                        if (operation.Intrinsic == Intrinsic.X86Cvtsi2si)
                         {
                             if (dest.Type == OperandType.I32)
                             {
@@ -538,7 +532,7 @@ namespace ARMeilleure.CodeGen.X86
                     if (src2.Kind == OperandKind.Constant)
                     {
                         offset = src2.AsInt32();
-                        index = null;
+                        index = default;
                     }
                     else
                     {
@@ -546,7 +540,7 @@ namespace ARMeilleure.CodeGen.X86
                         index = src2;
                     }
 
-                    MemoryOperand memOp = MemoryOp(dest.Type, src1, index, Multiplier.x1, offset);
+                    Operand memOp = MemoryOp(dest.Type, src1, index, Multiplier.x1, offset);
 
                     context.Assembler.Lea(dest, memOp, dest.Type);
                 }
@@ -720,7 +714,7 @@ namespace ARMeilleure.CodeGen.X86
 
             if (operation.SourcesCount == 5) // CompareAndSwap128 has 5 sources, compared to CompareAndSwap64/32's 3.
             {
-                MemoryOperand memOp = MemoryOp(OperandType.I64, src1);
+                Operand memOp = MemoryOp(OperandType.I64, src1);
 
                 context.Assembler.Cmpxchg16b(memOp);
             }
@@ -731,7 +725,7 @@ namespace ARMeilleure.CodeGen.X86
 
                 EnsureSameType(src2, src3);
 
-                MemoryOperand memOp = MemoryOp(src3.Type, src1);
+                Operand memOp = MemoryOp(src3.Type, src1);
 
                 context.Assembler.Cmpxchg(memOp, src3);
             }
@@ -745,7 +739,7 @@ namespace ARMeilleure.CodeGen.X86
 
             EnsureSameType(src2, src3);
 
-            MemoryOperand memOp = MemoryOp(src3.Type, src1);
+            Operand memOp = MemoryOp(src3.Type, src1);
 
             context.Assembler.Cmpxchg16(memOp, src3);
         }
@@ -758,7 +752,7 @@ namespace ARMeilleure.CodeGen.X86
 
             EnsureSameType(src2, src3);
 
-            MemoryOperand memOp = MemoryOp(src3.Type, src1);
+            Operand memOp = MemoryOp(src3.Type, src1);
 
             context.Assembler.Cmpxchg8(memOp, src3);
         }
@@ -954,7 +948,7 @@ namespace ARMeilleure.CodeGen.X86
 
             Operand rsp = Register(X86Register.Rsp);
 
-            MemoryOperand memOp = MemoryOp(dest.Type, rsp, null, Multiplier.x1, offs);
+            Operand memOp = MemoryOp(dest.Type, rsp, default, Multiplier.x1, offs);
 
             GenerateLoad(context, memOp, dest);
         }
@@ -1153,7 +1147,7 @@ namespace ARMeilleure.CodeGen.X86
 
             Operand rsp = Register(X86Register.Rsp);
 
-            MemoryOperand memOp = MemoryOp(source.Type, rsp, null, Multiplier.x1, offs);
+            Operand memOp = MemoryOp(source.Type, rsp, default, Multiplier.x1, offs);
 
             GenerateStore(context, memOp, source);
         }
@@ -1169,7 +1163,7 @@ namespace ARMeilleure.CodeGen.X86
 
             Operand rsp = Register(X86Register.Rsp);
 
-            MemoryOperand memOp = MemoryOp(OperandType.I64, rsp, null, Multiplier.x1, offs);
+            Operand memOp = MemoryOp(OperandType.I64, rsp, default, Multiplier.x1, offs);
 
             context.Assembler.Lea(dest, memOp, OperandType.I64);
         }
@@ -1644,19 +1638,19 @@ namespace ARMeilleure.CodeGen.X86
             context.Assembler.Pshufd(dest, dest, 0xfc);
         }
 
-        private static bool MatchOperation(Node node, Instruction inst, OperandType destType, Register destReg)
+        private static bool MatchOperation(Operation node, Instruction inst, OperandType destType, Register destReg)
         {
-            if (!(node is Operation operation) || node.DestinationsCount == 0)
+            if (node == default || node.DestinationsCount == 0)
             {
                 return false;
             }
 
-            if (operation.Instruction != inst)
+            if (node.Instruction != inst)
             {
                 return false;
             }
 
-            Operand dest = operation.Destination;
+            Operand dest = node.Destination;
 
             return dest.Kind == OperandKind.Register &&
                    dest.Type == destType &&
@@ -1761,7 +1755,7 @@ namespace ARMeilleure.CodeGen.X86
 
                 offset -= 16;
 
-                MemoryOperand memOp = MemoryOp(OperandType.V128, rsp, null, Multiplier.x1, offset);
+                Operand memOp = MemoryOp(OperandType.V128, rsp, default, Multiplier.x1, offset);
 
                 context.Assembler.Movdqu(memOp, Xmm((X86Register)bit));
 
@@ -1791,7 +1785,7 @@ namespace ARMeilleure.CodeGen.X86
 
                 offset -= 16;
 
-                MemoryOperand memOp = MemoryOp(OperandType.V128, rsp, null, Multiplier.x1, offset);
+                Operand memOp = MemoryOp(OperandType.V128, rsp, default, Multiplier.x1, offset);
 
                 context.Assembler.Movdqu(Xmm((X86Register)bit), memOp);
 
@@ -1832,17 +1826,17 @@ namespace ARMeilleure.CodeGen.X86
 
             for (int offset = PageSize; offset < size; offset += PageSize)
             {
-                Operand memOp = MemoryOp(OperandType.I32, rsp, null, Multiplier.x1, -offset);
+                Operand memOp = MemoryOp(OperandType.I32, rsp, default, Multiplier.x1, -offset);
 
                 context.Assembler.Mov(temp, memOp, OperandType.I32);
             }
         }
 
-        private static MemoryOperand Memory(Operand operand, OperandType type)
+        private static Operand Memory(Operand operand, OperandType type)
         {
             if (operand.Kind == OperandKind.Memory)
             {
-                return operand as MemoryOperand;
+                return operand;
             }
 
             return MemoryOp(type, operand);
@@ -1850,12 +1844,12 @@ namespace ARMeilleure.CodeGen.X86
 
         private static Operand Register(X86Register register, OperandType type = OperandType.I64)
         {
-            return OperandHelper.Register((int)register, RegisterType.Integer, type);
+            return Operand.Factory.Register((int)register, RegisterType.Integer, type);
         }
 
         private static Operand Xmm(X86Register register)
         {
-            return OperandHelper.Register((int)register, RegisterType.Vector, OperandType.V128);
+            return Operand.Factory.Register((int)register, RegisterType.Vector, OperandType.V128);
         }
     }
 }

ARMeilleure/CodeGen/X86/X86Optimizer.cs

@@ -2,9 +2,8 @@
 using ARMeilleure.IntermediateRepresentation;
 using ARMeilleure.Translation;
 using System.Collections.Generic;
-
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operation.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperationHelper;
 
 namespace ARMeilleure.CodeGen.X86
 {
@@ -34,32 +33,27 @@ namespace ARMeilleure.CodeGen.X86
             {
                 constants.Clear();
 
-                Node nextNode;
+                Operation nextNode;
 
-                for (Node node = block.Operations.First; node != null; node = nextNode)
+                for (Operation node = block.Operations.First; node != default; node = nextNode)
                 {
                     nextNode = node.ListNext;
 
-                    if (!(node is Operation operation))
-                    {
-                        continue;
-                    }
-
                     // Insert copies for constants that can't fit on a 32-bits immediate.
                     // Doing this early unblocks a few optimizations.
-                    if (operation.Instruction == Instruction.Add)
+                    if (node.Instruction == Instruction.Add)
                     {
-                        Operand src1 = operation.GetSource(0);
+                        Operand src1 = node.GetSource(0);
-                        Operand src2 = operation.GetSource(1);
+                        Operand src2 = node.GetSource(1);
 
                         if (src1.Kind == OperandKind.Constant && (src1.Relocatable || CodeGenCommon.IsLongConst(src1)))
                         {
-                            operation.SetSource(0, GetConstantCopy(block, operation, src1));
+                            node.SetSource(0, GetConstantCopy(block, node, src1));
                         }
 
                         if (src2.Kind == OperandKind.Constant && (src2.Relocatable || CodeGenCommon.IsLongConst(src2)))
                         {
-                            operation.SetSource(1, GetConstantCopy(block, operation, src2));
+                            node.SetSource(1, GetConstantCopy(block, node, src2));
                         }
                     }
 
@@ -70,24 +64,24 @@ namespace ARMeilleure.CodeGen.X86
                     //  mov rax, [rax]
                     // Into:
                     //  mov rax, [rax+rbx*4+0xcafe]
-                    if (IsMemoryLoadOrStore(operation.Instruction))
+                    if (IsMemoryLoadOrStore(node.Instruction))
                     {
                         OperandType type;
 
-                        if (operation.Destination != null)
+                        if (node.Destination != default)
                         {
-                            type = operation.Destination.Type;
+                            type = node.Destination.Type;
                         }
                         else
                         {
-                            type = operation.GetSource(1).Type;
+                            type = node.GetSource(1).Type;
                         }
 
-                        MemoryOperand memOp = GetMemoryOperandOrNull(operation.GetSource(0), type);
+                        Operand memOp = GetMemoryOperandOrNull(node.GetSource(0), type);
 
-                        if (memOp != null)
+                        if (memOp != default)
                         {
-                            operation.SetSource(0, memOp);
+                            node.SetSource(0, memOp);
                         }
                     }
                 }
@@ -96,7 +90,7 @@ namespace ARMeilleure.CodeGen.X86
             Optimizer.RemoveUnusedNodes(cfg);
         }
 
-        private static MemoryOperand GetMemoryOperandOrNull(Operand addr, OperandType type)
+        private static Operand GetMemoryOperandOrNull(Operand addr, OperandType type)
         {
             Operand baseOp = addr;
 
@@ -117,10 +111,10 @@ namespace ARMeilleure.CodeGen.X86
             // If baseOp is still equal to address, then there's nothing that can be optimized.
             if (baseOp == addr)
             {
-                return null;
+                return default;
             }
 
-            if (imm == 0 && scale == Multiplier.x1 && indexOp != null)
+            if (imm == 0 && scale == Multiplier.x1 && indexOp != default)
             {
                 imm = GetConstOp(ref indexOp);
             }
@@ -132,7 +126,7 @@ namespace ARMeilleure.CodeGen.X86
         {
             Operation operation = GetAsgOpWithInst(baseOp, Instruction.Add);
 
-            if (operation == null)
+            if (operation == default)
             {
                 return 0;
             }
@@ -172,13 +166,13 @@ namespace ARMeilleure.CodeGen.X86
 
         private static (Operand, Multiplier) GetIndexOp(ref Operand baseOp)
         {
-            Operand indexOp = null;
+            Operand indexOp = default;
 
             Multiplier scale = Multiplier.x1;
 
             Operation addOp = GetAsgOpWithInst(baseOp, Instruction.Add);
 
-            if (addOp == null)
+            if (addOp == default)
             {
                 return (indexOp, scale);
             }
@@ -198,14 +192,14 @@ namespace ARMeilleure.CodeGen.X86
 
             bool indexOnSrc2 = false;
 
-            if (shlOp == null)
+            if (shlOp == default)
             {
                 shlOp = GetAsgOpWithInst(src2, Instruction.ShiftLeft);
 
                 indexOnSrc2 = true;
             }
 
-            if (shlOp != null)
+            if (shlOp != default)
             {
                 Operand shSrc = shlOp.GetSource(0);
                 Operand shift = shlOp.GetSource(1);
@@ -233,24 +227,19 @@ namespace ARMeilleure.CodeGen.X86
             // If we have multiple assignments, folding is not safe
             // as the value may be different depending on the
             // control flow path.
-            if (op.Assignments.Count != 1)
+            if (op.AssignmentsCount != 1)
             {
-                return null;
+                return default;
             }
 
-            Node asgOp = op.Assignments[0];
+            Operation asgOp = op.Assignments[0];
 
-            if (!(asgOp is Operation operation))
+            if (asgOp.Instruction != inst)
             {
-                return null;
+                return default;
             }
 
-            if (operation.Instruction != inst)
+            return asgOp;
-            {
-                return null;
-            }
-
-            return operation;
         }
 
         private static bool IsMemoryLoadOrStore(Instruction inst)

ARMeilleure/Common/AddressTable.cs

@@ -211,7 +211,7 @@ namespace ARMeilleure.Common
         private IntPtr Allocate<T>(int length, T fill, bool leaf) where T : unmanaged
         {
             var size = sizeof(T) * length;
-            var page = Marshal.AllocHGlobal(size);
+            var page = (IntPtr)NativeAllocator.Instance.Allocate((uint)size);
             var span = new Span<T>((void*)page, length);
 
             span.Fill(fill);

ARMeilleure/Common/Allocator.cs

@@ -0,0 +1,24 @@
+using System;
+
+namespace ARMeilleure.Common
+{
+    unsafe abstract class Allocator : IDisposable
+    {
+        public T* Allocate<T>(ulong count = 1) where T : unmanaged
+        {
+            return (T*)Allocate(count * (uint)sizeof(T));
+        }
+
+        public abstract void* Allocate(ulong size);
+
+        public abstract void Free(void* block);
+
+        protected virtual void Dispose(bool disposing) { }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+    }
+}

ARMeilleure/Common/ArenaAllocator.cs

@@ -0,0 +1,188 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Threading;
+
+namespace ARMeilleure.Common
+{
+    unsafe sealed class ArenaAllocator : Allocator
+    {
+        private class PageInfo
+        {
+            public byte* Pointer;
+            public byte Unused;
+            public int UnusedCounter;
+        }
+
+        private int _lastReset;
+        private ulong _index;
+        private int _pageIndex;
+        private PageInfo _page;
+        private List<PageInfo> _pages;
+        private readonly ulong _pageSize;
+        private readonly uint _pageCount;
+        private readonly List<IntPtr> _extras;
+
+        public ArenaAllocator(uint pageSize, uint pageCount)
+        {
+            _lastReset = Environment.TickCount;
+
+            // Set _index to pageSize so that the first allocation goes through the slow path.
+            _index = pageSize;
+            _pageIndex = -1;
+
+            _page = null;
+            _pages = new List<PageInfo>();
+            _pageSize = pageSize;
+            _pageCount = pageCount;
+
+            _extras = new List<IntPtr>();
+        }
+
+        public Span<T> AllocateSpan<T>(ulong count) where T : unmanaged
+        {
+            return new Span<T>(Allocate<T>(count), (int)count);
+        }
+
+        public override void* Allocate(ulong size)
+        {
+            if (_index + size <= _pageSize)
+            {
+                byte* result = _page.Pointer + _index;
+
+                _index += size;
+
+                return result;
+            }
+
+            return AllocateSlow(size);
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private void* AllocateSlow(ulong size)
+        {
+            if (size > _pageSize)
+            {
+                void* extra = NativeAllocator.Instance.Allocate(size);
+
+                _extras.Add((IntPtr)extra);
+
+                return extra;
+            }
+
+            if (_index + size > _pageSize)
+            {
+                _index = 0;
+                _pageIndex++;
+            }
+
+            if (_pageIndex < _pages.Count)
+            {
+                _page = _pages[_pageIndex];
+                _page.Unused = 0;
+            }
+            else
+            {
+                _page = new PageInfo();
+                _page.Pointer = (byte*)NativeAllocator.Instance.Allocate(_pageSize);
+
+                _pages.Add(_page);
+            }
+
+            byte* result = _page.Pointer + _index;
+
+            _index += size;
+
+            return result;
+        }
+
+        public override void Free(void* block) { }
+
+        public void Reset()
+        {
+            _index = _pageSize;
+            _pageIndex = -1;
+            _page = null;
+
+            // Free excess pages that was allocated.
+            while (_pages.Count > _pageCount)
+            {
+                NativeAllocator.Instance.Free(_pages[_pages.Count - 1].Pointer);
+
+                _pages.RemoveAt(_pages.Count - 1);
+            }
+
+            // Free extra blocks that are not page-sized
+            foreach (IntPtr ptr in _extras)
+            {
+                NativeAllocator.Instance.Free((void*)ptr);
+            }
+
+            _extras.Clear();
+
+            // Free pooled pages that has not been used in a while. Remove pages at the back first, because we try to
+            // keep the pages at the front alive, since they're more likely to be hot and in the d-cache.
+            bool removing = true;
+
+            // If arena is used frequently, keep pages for longer. Otherwise keep pages for a shorter amount of time.
+            int now = Environment.TickCount;
+            int count = (now - _lastReset) switch {
+                >= 5000 => 0,
+                >= 2500 => 50,
+                >= 1000 => 100,
+                >= 10   => 1500,
+                _       => 5000
+            };
+
+            for (int i = _pages.Count - 1; i >= 0; i--)
+            {
+                PageInfo page = _pages[i];
+
+                if (page.Unused == 0)
+                {
+                    page.UnusedCounter = 0;
+                }
+
+                page.UnusedCounter += page.Unused;
+                page.Unused = 1;
+
+                // If page not used after `count` resets, remove it.
+                if (removing && page.UnusedCounter >= count)
+                {
+                    NativeAllocator.Instance.Free(page.Pointer);
+
+                    _pages.RemoveAt(i);
+                }
+                else
+                {
+                    removing = false;
+                }
+            }
+
+            _lastReset = now;
+        }
+
+        protected override void Dispose(bool disposing)
+        {
+            if (_pages != null)
+            {
+                foreach (PageInfo info in _pages)
+                {
+                    NativeAllocator.Instance.Free(info.Pointer);
+                }
+
+                foreach (IntPtr ptr in _extras)
+                {
+                    NativeAllocator.Instance.Free((void*)ptr);
+                }
+
+                _pages = null;
+            }
+        }
+
+        ~ArenaAllocator()
+        {
+            Dispose(false);
+        }
+    }
+}

ARMeilleure/Common/BitMap.cs

@@ -1,57 +1,27 @@
+using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Numerics;
 
 namespace ARMeilleure.Common
 {
-    class BitMap : IEnumerator<int>, IEnumerable<int>
+    unsafe class BitMap : IEnumerable<int>, IDisposable
     {
         private const int IntSize = 64;
         private const int IntMask = IntSize - 1;
 
-        private readonly List<long> _masks;
+        private int _count;
+        private long* _masks;
+        private readonly Allocator _allocator;
 
-        private int _enumIndex;
+        public BitMap(Allocator allocator)
-        private long _enumMask;
-        private int _enumBit;
-
-        public int Current => _enumIndex * IntSize + _enumBit;
-        object IEnumerator.Current => Current;
-
-        public BitMap()
         {
-            _masks = new List<long>(0);
+            _allocator = allocator;
         }
 
-        public BitMap(int initialCapacity)
+        public BitMap(Allocator allocator, int capacity) : this(allocator)
         {
-            int count = (initialCapacity + IntMask) / IntSize;
+            EnsureCapacity(capacity);
-
-            _masks = new List<long>(count);
-
-            while (count-- > 0)
-            {
-                _masks.Add(0);
-            }
-        }
-
-        public BitMap Reset(int initialCapacity)
-        {
-            int count = (initialCapacity + IntMask) / IntSize;
-
-            if (count > _masks.Capacity)
-            {
-                _masks.Capacity = count;
-            }
-
-            _masks.Clear();
-
-            while (count-- > 0)
-            {
-                _masks.Add(0);
-            }
-
-            return this;
         }
 
         public bool Set(int bit)
@@ -97,7 +67,7 @@ namespace ARMeilleure.Common
 
         public int FindFirstUnset()
         {
-            for (int index = 0; index < _masks.Count; index++)
+            for (int index = 0; index < _count; index++)
             {
                 long mask = _masks[index];
 
@@ -107,16 +77,16 @@ namespace ARMeilleure.Common
                 }
             }
 
-            return _masks.Count * IntSize;
+            return _count * IntSize;
         }
 
         public bool Set(BitMap map)
         {
-            EnsureCapacity(map._masks.Count * IntSize);
+            EnsureCapacity(map._count * IntSize);
 
             bool modified = false;
 
-            for (int index = 0; index < _masks.Count; index++)
+            for (int index = 0; index < _count; index++)
             {
                 long newValue = _masks[index] | map._masks[index];
 
@@ -133,11 +103,11 @@ namespace ARMeilleure.Common
 
         public bool Clear(BitMap map)
         {
-            EnsureCapacity(map._masks.Count * IntSize);
+            EnsureCapacity(map._count * IntSize);
 
             bool modified = false;
 
-            for (int index = 0; index < _masks.Count; index++)
+            for (int index = 0; index < _count; index++)
             {
                 long newValue = _masks[index] & ~map._masks[index];
 
@@ -152,15 +122,34 @@ namespace ARMeilleure.Common
             return modified;
         }
 
-        #region IEnumerable<long> Methods
-
-        // Note: The bit enumerator is embedded in this class to avoid creating garbage when enumerating.
-
         private void EnsureCapacity(int size)
         {
-            while (_masks.Count * IntSize < size)
+            int count = (size + IntMask) / IntSize;
+
+            if (count > _count)
             {
-                _masks.Add(0);
+                var oldMask = _masks;
+                var oldSpan = new Span<long>(_masks, _count);
+
+                _masks = _allocator.Allocate<long>((uint)count);
+                _count = count;
+
+                var newSpan = new Span<long>(_masks, _count);
+
+                oldSpan.CopyTo(newSpan);
+                newSpan.Slice(oldSpan.Length).Clear();
+
+                _allocator.Free(oldMask);
+            }
+        }
+
+        public void Dispose()
+        {
+            if (_masks != null)
+            {
+                _allocator.Free(_masks);
+
+                _masks = null;
             }
         }
 
@@ -169,39 +158,59 @@ namespace ARMeilleure.Common
             return GetEnumerator();
         }
 
-        public IEnumerator<int> GetEnumerator()
+        IEnumerator<int> IEnumerable<int>.GetEnumerator()
         {
-            Reset();
+            return GetEnumerator();
-            return this;
+        }
+
+        public Enumerator GetEnumerator()
+        {
+            return new Enumerator(this);
+        }
+
+        public struct Enumerator : IEnumerator<int>
+        {
+            private int _index;
+            private long _mask;
+            private int _bit;
+            private readonly BitMap _map;
+
+            public int Current => _index * IntSize + _bit;
+            object IEnumerator.Current => Current;
+
+            public Enumerator(BitMap map)
+            {
+                _index = -1;
+                _mask = 0;
+                _bit = 0;
+                _map = map;
             }
 
             public bool MoveNext()
             {
-            if (_enumMask != 0)
+                if (_mask != 0)
                 {
-                _enumMask &= ~(1L << _enumBit);
+                    _mask &= ~(1L << _bit);
                 }
-            while (_enumMask == 0)
+
+                while (_mask == 0)
                 {
-                if (++_enumIndex >= _masks.Count)
+                    if (++_index >= _map._count)
                     {
                         return false;
                     }
-                _enumMask = _masks[_enumIndex];
+
+                    _mask = _map._masks[_index];
                 }
-            _enumBit = BitOperations.TrailingZeroCount(_enumMask);
+
+                _bit = BitOperations.TrailingZeroCount(_mask);
+
                 return true;
             }
 
-        public void Reset()
+            public void Reset() { }
-        {
-            _enumIndex = -1;
-            _enumMask = 0;
-            _enumBit = 0;
-        }
 
             public void Dispose() { }
-
+        }
-        #endregion
     }
 }

ARMeilleure/Common/BitMapPool.cs

@@ -1,32 +0,0 @@
-namespace ARMeilleure.Common
-{
-    static class BitMapPool
-    {
-        public static BitMap Allocate(int initialCapacity)
-        {
-            return BitMap().Reset(initialCapacity);
-        }
-
-        #region "ThreadStaticPool"
-        public static void PrepareBitMapPool(int groupId = 0)
-        {
-            ThreadStaticPool<BitMap>.PreparePool(groupId, ChunkSizeLimit.Small);
-        }
-
-        private static BitMap BitMap()
-        {
-            return ThreadStaticPool<BitMap>.Instance.Allocate();
-        }
-
-        public static void ResetBitMapPool(int groupId = 0)
-        {
-            ThreadStaticPool<BitMap>.ResetPool(groupId);
-        }
-
-        public static void DisposeBitMapPools()
-        {
-            ThreadStaticPool<BitMap>.DisposePools();
-        }
-        #endregion
-    }
-}

ARMeilleure/Common/EntryTable.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Numerics;
-using System.Runtime.InteropServices;
 
 namespace ARMeilleure.Common
 {
@@ -41,7 +40,7 @@ namespace ARMeilleure.Common
                 throw new ArgumentException("Size of TEntry cannot be zero.");
             }
 
-            _allocated = new BitMap();
+            _allocated = new BitMap(NativeAllocator.Instance);
             _pages = new Dictionary<int, IntPtr>();
             _pageLogCapacity = BitOperations.Log2((uint)(pageSize / sizeof(TEntry)));
             _pageCapacity = 1 << _pageLogCapacity;
@@ -150,7 +149,7 @@ namespace ARMeilleure.Common
 
             if (!_pages.TryGetValue(pageIndex, out IntPtr page))
             {
-                page = Marshal.AllocHGlobal(sizeof(TEntry) * _pageCapacity);
+                page = (IntPtr)NativeAllocator.Instance.Allocate((uint)sizeof(TEntry) * (uint)_pageCapacity);
 
                 _pages.Add(pageIndex, page);
             }
@@ -172,13 +171,15 @@ namespace ARMeilleure.Common
         /// instance.
         /// </summary>
         /// <param name="disposing"><see langword="true"/> to dispose managed resources also; otherwise just unmanaged resouces</param>
-        protected virtual void Dispose(bool disposing)
+        protected unsafe virtual void Dispose(bool disposing)
         {
             if (!_disposed)
             {
+                _allocated.Dispose();
+
                 foreach (var page in _pages.Values)
                 {
-                    Marshal.FreeHGlobal(page);
+                    NativeAllocator.Instance.Free((void*)page);
                 }
 
                 _disposed = true;

ARMeilleure/Common/NativeAllocator.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace ARMeilleure.Common
+{
+    unsafe sealed class NativeAllocator : Allocator
+    {
+        public static NativeAllocator Instance { get; } = new();
+
+        public override void* Allocate(ulong size)
+        {
+            void* result = (void*)Marshal.AllocHGlobal((IntPtr)size);
+
+            if (result == null)
+            {
+                throw new OutOfMemoryException();
+            }
+
+            return result;
+        }
+
+        public override void Free(void* block)
+        {
+            Marshal.FreeHGlobal((IntPtr)block);
+        }
+    }
+}

ARMeilleure/Common/ThreadStaticPool.cs

@@ -1,219 +0,0 @@
-using ARMeilleure.Translation.PTC;
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-
-namespace ARMeilleure.Common
-{
-    class ThreadStaticPool<T> where T : class, new()
-    {
-        [ThreadStatic]
-        private static ThreadStaticPool<T> _instance;
-
-        public static ThreadStaticPool<T> Instance
-        {
-            get
-            {
-                if (_instance == null)
-                {
-                    PreparePool(); // So that we can still use a pool when blindly initializing one.
-                }
-
-                return _instance;
-            }
-        }
-
-        private static readonly ConcurrentDictionary<int, Stack<ThreadStaticPool<T>>> _pools = new();
-
-        private static Stack<ThreadStaticPool<T>> GetPools(int groupId)
-        {
-            return _pools.GetOrAdd(groupId, (groupId) => new());
-        }
-
-        public static void PreparePool(
-            int groupId = 0,
-            ChunkSizeLimit chunkSizeLimit = ChunkSizeLimit.Large,
-            PoolSizeIncrement poolSizeIncrement = PoolSizeIncrement.Default)
-        {
-            if (Ptc.State == PtcState.Disabled)
-            {
-                PreparePoolDefault(groupId, (int)chunkSizeLimit, (int)poolSizeIncrement);
-            }
-            else
-            {
-                PreparePoolSlim((int)chunkSizeLimit, (int)poolSizeIncrement);
-            }
-        }
-
-        private static void PreparePoolDefault(int groupId, int chunkSizeLimit, int poolSizeIncrement)
-        {
-            // Prepare the pool for this thread, ideally using an existing one from the specified group.
-
-            if (_instance == null)
-            {
-                var pools = GetPools(groupId);
-                lock (pools)
-                {
-                    _instance = (pools.Count != 0) ? pools.Pop() : new(chunkSizeLimit, poolSizeIncrement);
-                }
-            }
-        }
-
-        private static void PreparePoolSlim(int chunkSizeLimit, int poolSizeIncrement)
-        {
-            // Prepare the pool for this thread.
-
-            if (_instance == null)
-            {
-                _instance = new(chunkSizeLimit, poolSizeIncrement);
-            }
-        }
-
-        public static void ResetPool(int groupId = 0)
-        {
-            if (Ptc.State == PtcState.Disabled)
-            {
-                ResetPoolDefault(groupId);
-            }
-            else
-            {
-                ResetPoolSlim();
-            }
-        }
-
-        private static void ResetPoolDefault(int groupId)
-        {
-            // Reset, limit if necessary, and return the pool for this thread to the specified group.
-
-            if (_instance != null)
-            {
-                var pools = GetPools(groupId);
-                lock (pools)
-                {
-                    _instance.Clear();
-                    _instance.ChunkSizeLimiter();
-                    pools.Push(_instance);
-
-                    _instance = null;
-                }
-            }
-        }
-
-        private static void ResetPoolSlim()
-        {
-            // Reset, limit if necessary, the pool for this thread.
-
-            if (_instance != null)
-            {
-                _instance.Clear();
-                _instance.ChunkSizeLimiter();
-            }
-        }
-
-        public static void DisposePools()
-        {
-            if (Ptc.State == PtcState.Disabled)
-            {
-                DisposePoolsDefault();
-            }
-            else
-            {
-                DisposePoolSlim();
-            }
-        }
-
-        private static void DisposePoolsDefault()
-        {
-            // Resets any static references to the pools used by threads for each group, allowing them to be garbage collected.
-
-            foreach (var pools in _pools.Values)
-            {
-                foreach (var instance in pools)
-                {
-                    instance.Dispose();
-                }
-
-                pools.Clear();
-            }
-
-            _pools.Clear();
-        }
-
-        private static void DisposePoolSlim()
-        {
-            // Dispose the pool for this thread.
-
-            if (_instance != null)
-            {
-                _instance.Dispose();
-
-                _instance = null;
-            }
-        }
-
-        private List<T[]> _pool;
-        private int _chunkIndex = -1;
-        private int _poolIndex = -1;
-        private int _chunkSizeLimit;
-        private int _poolSizeIncrement;
-
-        private ThreadStaticPool(int chunkSizeLimit, int poolSizeIncrement)
-        {
-            _chunkSizeLimit = chunkSizeLimit;
-            _poolSizeIncrement = poolSizeIncrement;
-
-            _pool = new(chunkSizeLimit * 2);
-
-            AddChunkIfNeeded();
-        }
-
-        public T Allocate()
-        {
-            if (++_poolIndex >= _poolSizeIncrement)
-            {
-                AddChunkIfNeeded();
-
-                _poolIndex = 0;
-            }
-
-            return _pool[_chunkIndex][_poolIndex];
-        }
-
-        private void AddChunkIfNeeded()
-        {
-            if (++_chunkIndex >= _pool.Count)
-            {
-                T[] pool = new T[_poolSizeIncrement];
-
-                for (int i = 0; i < _poolSizeIncrement; i++)
-                {
-                    pool[i] = new T();
-                }
-
-                _pool.Add(pool);
-            }
-        }
-
-        public void Clear()
-        {
-            _chunkIndex = 0;
-            _poolIndex = -1;
-        }
-
-        private void ChunkSizeLimiter()
-        {
-            if (_pool.Count >= _chunkSizeLimit)
-            {
-                int newChunkSize = _chunkSizeLimit / 2;
-
-                _pool.RemoveRange(newChunkSize, _pool.Count - newChunkSize);
-                _pool.Capacity = _chunkSizeLimit * 2;
-            }
-        }
-
-        private void Dispose()
-        {
-            _pool = null;
-        }
-    }
-}

ARMeilleure/Common/ThreadStaticPoolEnums.cs

@@ -1,14 +0,0 @@
-namespace ARMeilleure.Common
-{
-    public enum PoolSizeIncrement
-    {
-        Default = 200
-    }
-
-    public enum ChunkSizeLimit
-    {
-        Large = 200000 / PoolSizeIncrement.Default,
-        Medium = 100000 / PoolSizeIncrement.Default,
-        Small = 50000 / PoolSizeIncrement.Default
-    }
-}

ARMeilleure/Diagnostics/IRDumper.cs

@@ -18,7 +18,7 @@ namespace ARMeilleure.Diagnostics
         private readonly Dictionary<Operand, string> _localNames;
         private readonly Dictionary<ulong, string> _symbolNames;
 
-        private IRDumper(int indent)
+        public IRDumper(int indent)
         {
             _indentLevel = indent;
 
@@ -62,15 +62,15 @@ namespace ARMeilleure.Diagnostics
                 _builder.Append(" cold");
             }
 
-            if (block.SuccessorCount > 0)
+            if (block.SuccessorsCount > 0)
             {
                 _builder.Append(" (");
 
-                for (int i = 0; i < block.SuccessorCount; i++)
+                for (int i = 0; i < block.SuccessorsCount; i++)
                 {
                     DumpBlockName(block.GetSuccessor(i));
 
-                    if (i < block.SuccessorCount - 1)
+                    if (i < block.SuccessorsCount - 1)
                     {
                         _builder.Append(", ");
                     }
@@ -84,7 +84,7 @@ namespace ARMeilleure.Diagnostics
 
         private void DumpOperand(Operand operand)
         {
-            if (operand == null)
+            if (operand == default)
             {
                 _builder.Append("<NULL>");
                 return;
@@ -131,13 +131,13 @@ namespace ARMeilleure.Diagnostics
                     break;
 
                 case OperandKind.Memory:
-                    var memOp = (MemoryOperand)operand;
+                    var memOp = operand.GetMemory();
 
                     _builder.Append('[');
 
                     DumpOperand(memOp.BaseAddress);
 
-                    if (memOp.Index != null)
+                    if (memOp.Index != default)
                     {
                         _builder.Append(" + ");
 
@@ -165,7 +165,7 @@ namespace ARMeilleure.Diagnostics
             }
         }
 
-        private void DumpNode(Node node)
+        private void DumpNode(ControlFlowGraph cfg, Operation node)
         {
             for (int index = 0; index < node.DestinationsCount; index++)
             {
@@ -183,14 +183,18 @@ namespace ARMeilleure.Diagnostics
 
             switch (node)
             {
-                case PhiNode phi:
+                case Operation operation:
+                    if (operation.Instruction == Instruction.Phi)
+                    {
+                        PhiOperation phi = operation.AsPhi();
+
                         _builder.Append("Phi ");
 
                         for (int index = 0; index < phi.SourcesCount; index++)
                         {
                             _builder.Append('(');
 
-                        DumpBlockName(phi.GetBlock(index));
+                            DumpBlockName(phi.GetBlock(cfg, index));
 
                             _builder.Append(": ");
 
@@ -203,18 +207,17 @@ namespace ARMeilleure.Diagnostics
                                 _builder.Append(", ");
                             }
                         }
-                    break;
 
-                case Operation operation:
+                        break;
+                    }
+
                     bool comparison = false;
 
                     _builder.Append(operation.Instruction);
 
                     if (operation.Instruction == Instruction.Extended)
                     {
-                        var intrinOp = (IntrinsicOperation)operation;
+                        _builder.Append('.').Append(operation.Intrinsic);
-
-                        _builder.Append('.').Append(intrinOp.Intrinsic);
                     }
                     else if (operation.Instruction == Instruction.BranchIf ||
                              operation.Instruction == Instruction.Compare)
@@ -277,10 +280,10 @@ namespace ARMeilleure.Diagnostics
 
                 dumper.IncreaseIndentation();
 
-                for (Node node = block.Operations.First; node != null; node = node.ListNext)
+                for (Operation node = block.Operations.First; node != default; node = node.ListNext)
                 {
                     dumper.Indent();
-                    dumper.DumpNode(node);
+                    dumper.DumpNode(cfg, node);
 
                     dumper._builder.AppendLine();
                 }

ARMeilleure/Instructions/InstEmitAlu.cs

@@ -6,7 +6,7 @@ using System.Diagnostics;
 
 using static ARMeilleure.Instructions.InstEmitAluHelper;
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitAlu32.cs

@@ -5,7 +5,7 @@ using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitAluHelper;
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitAluHelper.cs

@@ -6,7 +6,7 @@ using System;
 using System.Diagnostics;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitBfm.cs

@@ -3,7 +3,7 @@ using ARMeilleure.IntermediateRepresentation;
 using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitCcmp.cs

@@ -6,7 +6,7 @@ using ARMeilleure.Translation;
 using static ARMeilleure.Instructions.InstEmitAluHelper;
 using static ARMeilleure.Instructions.InstEmitFlowHelper;
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitCsel.cs

@@ -4,7 +4,7 @@ using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitFlowHelper;
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitDiv.cs

@@ -3,7 +3,7 @@ using ARMeilleure.IntermediateRepresentation;
 using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitException.cs

@@ -1,7 +1,7 @@
 using ARMeilleure.Decoders;
 using ARMeilleure.Translation;
 
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitException32.cs

@@ -2,7 +2,7 @@
 using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitFlowHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitFlow.cs

@@ -5,7 +5,7 @@ using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitFlowHelper;
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitFlow32.cs

@@ -5,7 +5,7 @@ using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitFlowHelper;
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitFlowHelper.cs

@@ -6,7 +6,7 @@ using ARMeilleure.Translation.Cache;
 using ARMeilleure.Translation.PTC;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -28,7 +28,7 @@ namespace ARMeilleure.Instructions
         {
             Operand cmpResult = context.TryGetComparisonResult(condition);
 
-            if (cmpResult != null)
+            if (cmpResult != default)
             {
                 return cmpResult;
             }

ARMeilleure/Instructions/InstEmitHashHelper.cs

@@ -5,7 +5,7 @@ using ARMeilleure.Translation;
 using System;
 using System.Diagnostics;
 
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
 
 namespace ARMeilleure.Instructions

ARMeilleure/Instructions/InstEmitHelper.cs

@@ -4,7 +4,7 @@ using ARMeilleure.State;
 using ARMeilleure.Translation;
 using System;
 
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitMemory.cs

@@ -4,7 +4,7 @@ using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitMemoryHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -121,7 +121,7 @@ namespace ARMeilleure.Instructions
 
         private static Operand GetAddress(ArmEmitterContext context, long addend = 0)
         {
-            Operand address = null;
+            Operand address = default;
 
             switch (context.CurrOp)
             {

ARMeilleure/Instructions/InstEmitMemory32.cs

@@ -6,7 +6,7 @@ using System;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitMemoryHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -156,7 +156,7 @@ namespace ARMeilleure.Instructions
             Operand n = context.Copy(GetIntA32(context, op.Rn));
             Operand m = GetMemM(context, setCarry: false);
 
-            Operand temp = null;
+            Operand temp = default;
 
             if (op.Index || op.WBack)
             {

ARMeilleure/Instructions/InstEmitMemoryEx.cs

@@ -6,7 +6,7 @@ using System.Diagnostics;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitMemoryExHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitMemoryEx32.cs

@@ -5,7 +5,7 @@ using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitMemoryExHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitMemoryExHelper.cs

@@ -3,7 +3,7 @@ using ARMeilleure.State;
 using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -20,7 +20,7 @@ namespace ARMeilleure.Instructions
                 if (size == 4)
                 {
                     // Only 128-bit CAS is guaranteed to have a atomic load.
-                    Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: false, 4);
+                    Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, default, write: false, 4);
 
                     Operand zero = context.VectorZero();
 
@@ -109,7 +109,7 @@ namespace ARMeilleure.Instructions
                 context.BranchIfTrue(lblExit, exFailed);
 
                 // STEP 2: We have exclusive access and the address is valid, attempt the store using CAS.
-                Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: true, size);
+                Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, default, write: true, size);
 
                 Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset()));
                 Operand exValue = size switch

ARMeilleure/Instructions/InstEmitMemoryHelper.cs

@@ -7,7 +7,7 @@ using System;
 using System.Reflection;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -130,7 +130,7 @@ namespace ARMeilleure.Instructions
 
             Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false, size);
 
-            Operand value = null;
+            Operand value = default;
 
             switch (size)
             {
@@ -161,7 +161,7 @@ namespace ARMeilleure.Instructions
                 throw new ArgumentOutOfRangeException(nameof(size));
             }
 
-            Operand physAddr = EmitPtPointerLoad(context, address, null, write: false, size);
+            Operand physAddr = EmitPtPointerLoad(context, address, default, write: false, size);
 
             return size switch
             {
@@ -186,7 +186,7 @@ namespace ARMeilleure.Instructions
 
             Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false, size);
 
-            Operand value = null;
+            Operand value = default;
 
             switch (size)
             {
@@ -257,7 +257,7 @@ namespace ARMeilleure.Instructions
                 throw new ArgumentOutOfRangeException(nameof(size));
             }
 
-            Operand physAddr = EmitPtPointerLoad(context, address, null, write: true, size);
+            Operand physAddr = EmitPtPointerLoad(context, address, default, write: true, size);
 
             if (size < 3 && value.Type == OperandType.I64)
             {
@@ -348,7 +348,7 @@ namespace ARMeilleure.Instructions
             // If the VA is out of range, or not aligned to the access size, force PTE to 0 by masking it.
             pte = context.BitwiseAnd(pte, context.ShiftRightSI(context.Add(addrShifted, Const(-(long)ptLevelSize)), Const(63)));
 
-            if (lblSlowPath != null)
+            if (lblSlowPath != default)
             {
                 if (write)
                 {
@@ -505,7 +505,7 @@ namespace ARMeilleure.Instructions
                 case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128)); break;
             }
 
-            Operand value = null;
+            Operand value = default;
 
             if (size < 4)
             {

ARMeilleure/Instructions/InstEmitMove.cs

@@ -3,7 +3,7 @@ using ARMeilleure.IntermediateRepresentation;
 using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitMul32.cs

@@ -6,7 +6,7 @@ using System;
 
 using static ARMeilleure.Instructions.InstEmitAluHelper;
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitSimdArithmetic.cs

@@ -11,7 +11,7 @@ using System.Diagnostics;
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper32;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -126,10 +126,10 @@ namespace ARMeilleure.Instructions
                 8  => Clz_V_I8 (context, GetVec(op.Rn)),
                 16 => Clz_V_I16(context, GetVec(op.Rn)),
                 32 => Clz_V_I32(context, GetVec(op.Rn)),
-                _  => null
+                _  => default
             };
 
-            if (res != null)
+            if (res != default)
             {
                 if (op.RegisterSize == RegisterSize.Simd64)
                 {
@@ -159,7 +159,7 @@ namespace ARMeilleure.Instructions
         {
             if (!Optimizations.UseSsse3)
             {
-                return null;
+                return default;
             }
 
             // CLZ nibble table.
@@ -189,7 +189,7 @@ namespace ARMeilleure.Instructions
         {
             if (!Optimizations.UseSsse3)
             {
-                return null;
+                return default;
             }
 
             Operand maskSwap = X86GetElements(context, 0x80_0f_80_0d_80_0b_80_09, 0x80_07_80_05_80_03_80_01);
@@ -215,7 +215,7 @@ namespace ARMeilleure.Instructions
             // TODO: Use vplzcntd when AVX-512 is supported.
             if (!Optimizations.UseSse2)
             {
-                return null;
+                return default;
             }
 
             Operand AddVectorI32(Operand op0, Operand op1)      => context.AddIntrinsic(Intrinsic.X86Paddd, op0, op1);
@@ -3684,8 +3684,8 @@ namespace ARMeilleure.Instructions
                 Operand mask2 = context.AddIntrinsic(Intrinsic.X86Pand,  opF,   qMask);
                         mask2 = context.AddIntrinsic(Intrinsic.X86Cmpps, mask2, qMask, Const((int)CmpCondition.Equal));
 
-                qNaNMask = isQNaN == null ||  (bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andps,  mask2, mask1) : null;
+                qNaNMask = isQNaN == null ||  (bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andps,  mask2, mask1) : default;
-                sNaNMask = isQNaN == null || !(bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andnps, mask2, mask1) : null;
+                sNaNMask = isQNaN == null || !(bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andnps, mask2, mask1) : default;
             }
             else /* if ((op.Size & 1) == 1) */
             {
@@ -3698,8 +3698,8 @@ namespace ARMeilleure.Instructions
                 Operand mask2 = context.AddIntrinsic(Intrinsic.X86Pand,  opF,   qMask);
                         mask2 = context.AddIntrinsic(Intrinsic.X86Cmppd, mask2, qMask, Const((int)CmpCondition.Equal));
 
-                qNaNMask = isQNaN == null ||  (bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andpd,  mask2, mask1) : null;
+                qNaNMask = isQNaN == null ||  (bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andpd,  mask2, mask1) : default;
-                sNaNMask = isQNaN == null || !(bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andnpd, mask2, mask1) : null;
+                sNaNMask = isQNaN == null || !(bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andnpd, mask2, mask1) : default;
             }
         }
 
@@ -3707,11 +3707,11 @@ namespace ARMeilleure.Instructions
             ArmEmitterContext context,
             Func2I emit,
             bool scalar,
-            Operand n = null,
+            Operand n = default,
-            Operand m = null)
+            Operand m = default)
         {
-            Operand nCopy = n ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn));
+            Operand nCopy = n == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn)) : n;
-            Operand mCopy = m ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm));
+            Operand mCopy = m == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm)) : m;
 
             EmitSse2VectorIsNaNOpF(context, nCopy, out Operand nQNaNMask, out Operand nSNaNMask);
             EmitSse2VectorIsNaNOpF(context, mCopy, out _, out Operand mSNaNMask, isQNaN: false);
@@ -3734,7 +3734,7 @@ namespace ARMeilleure.Instructions
 
                 Operand res = context.AddIntrinsic(Intrinsic.X86Blendvps, resNaN, emit(nCopy, mCopy), resMask);
 
-                if (n != null || m != null)
+                if (n != default || m != default)
                 {
                     return res;
                 }
@@ -3750,7 +3750,7 @@ namespace ARMeilleure.Instructions
 
                 context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rd), res);
 
-                return null;
+                return default;
             }
             else /* if (sizeF == 1) */
             {
@@ -3768,7 +3768,7 @@ namespace ARMeilleure.Instructions
 
                 Operand res = context.AddIntrinsic(Intrinsic.X86Blendvpd, resNaN, emit(nCopy, mCopy), resMask);
 
-                if (n != null || m != null)
+                if (n != default || m != default)
                 {
                     return res;
                 }
@@ -3780,7 +3780,7 @@ namespace ARMeilleure.Instructions
 
                 context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rd), res);
 
-                return null;
+                return default;
             }
         }
 
@@ -3788,11 +3788,11 @@ namespace ARMeilleure.Instructions
             ArmEmitterContext context,
             Func2I emit,
             bool scalar,
-            Operand n = null,
+            Operand n = default,
-            Operand m = null)
+            Operand m = default)
         {
-            Operand nCopy = n ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn));
+            Operand nCopy = n == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn)) : n;
-            Operand mCopy = m ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm));
+            Operand mCopy = m == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm)) : m;
 
             EmitSseOrAvxEnterFtzAndDazModesOpF(context, out Operand isTrue);
 
@@ -3800,7 +3800,7 @@ namespace ARMeilleure.Instructions
 
             EmitSseOrAvxExitFtzAndDazModesOpF(context, isTrue);
 
-            if (n != null || m != null)
+            if (n != default || m != default)
             {
                 return res;
             }
@@ -3828,7 +3828,7 @@ namespace ARMeilleure.Instructions
 
             context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rd), res);
 
-            return null;
+            return default;
         }
 
         private static Operand EmitSse2VectorMaxMinOpF(ArmEmitterContext context, Operand n, Operand m, bool isMax)
@@ -3865,11 +3865,11 @@ namespace ARMeilleure.Instructions
             ArmEmitterContext context,
             bool isMaxNum,
             bool scalar,
-            Operand n = null,
+            Operand n = default,
-            Operand m = null)
+            Operand m = default)
         {
-            Operand nCopy = n ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn));
+            Operand nCopy = n == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn)) : n;
-            Operand mCopy = m ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm));
+            Operand mCopy = m == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm)) : m;
 
             EmitSse2VectorIsNaNOpF(context, nCopy, out Operand nQNaNMask, out _, isQNaN: true);
             EmitSse2VectorIsNaNOpF(context, mCopy, out Operand mQNaNMask, out _, isQNaN: true);
@@ -3896,7 +3896,7 @@ namespace ARMeilleure.Instructions
                     }, scalar: scalar, op1, op2);
                 }, scalar: scalar, nCopy, mCopy);
 
-                if (n != null || m != null)
+                if (n != default || m != default)
                 {
                     return res;
                 }
@@ -3912,7 +3912,7 @@ namespace ARMeilleure.Instructions
 
                 context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rd), res);
 
-                return null;
+                return default;
             }
             else /* if (sizeF == 1) */
             {
@@ -3934,7 +3934,7 @@ namespace ARMeilleure.Instructions
                     }, scalar: scalar, op1, op2);
                 }, scalar: scalar, nCopy, mCopy);
 
-                if (n != null || m != null)
+                if (n != default || m != default)
                 {
                     return res;
                 }
@@ -3946,7 +3946,7 @@ namespace ARMeilleure.Instructions
 
                 context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rd), res);
 
-                return null;
+                return default;
             }
         }
 

ARMeilleure/Instructions/InstEmitSimdArithmetic32.cs

@@ -8,7 +8,7 @@ using static ARMeilleure.Instructions.InstEmitFlowHelper;
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper32;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -1275,7 +1275,8 @@ namespace ARMeilleure.Instructions
         {
             OpCode32SimdSel op = (OpCode32SimdSel)context.CurrOp;
 
-            Operand condition = null;
+            Operand condition = default;
+
             switch (op.Cc)
             {
                 case OpCode32SimdSelMode.Eq:

ARMeilleure/Instructions/InstEmitSimdCmp.cs

@@ -6,7 +6,7 @@ using System;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitSimdCmp32.cs

@@ -8,7 +8,7 @@ using System.Reflection;
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper32;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitSimdCvt.cs

@@ -8,7 +8,7 @@ using System.Reflection;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitSimdCvt32.cs

@@ -9,7 +9,7 @@ using System.Reflection;
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper32;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -391,7 +391,8 @@ namespace ARMeilleure.Instructions
                 Operand zero = context.VectorZero();
 
                 Operand nCmp;
-                Operand nIntOrLong2 = null;
+                Operand nIntOrLong2 = default;
+
                 if (!signed)
                 {
                     nCmp = context.AddIntrinsic(Intrinsic.X86Cmpss, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual));
@@ -441,7 +442,8 @@ namespace ARMeilleure.Instructions
                 Operand zero = context.VectorZero();
 
                 Operand nCmp;
-                Operand nIntOrLong2 = null;
+                Operand nIntOrLong2 = default;
+
                 if (!signed)
                 {
                     nCmp = context.AddIntrinsic(Intrinsic.X86Cmpsd, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual));
@@ -510,7 +512,8 @@ namespace ARMeilleure.Instructions
                     Operand fpMaxValMask = X86GetAllElements(context, 0x4F000000); // 2.14748365E9f (2147483648)
 
                     Operand nInt = context.AddIntrinsic(Intrinsic.X86Cvtps2dq, nRes);
-                    Operand nInt2 = null;
+                    Operand nInt2 = default;
+
                     if (!signed)
                     {
                         nRes = context.AddIntrinsic(Intrinsic.X86Subps, nRes, fpMaxValMask);
@@ -551,7 +554,8 @@ namespace ARMeilleure.Instructions
                     Operand fpMaxValMask = X86GetAllElements(context, 0x43E0000000000000L); // 9.2233720368547760E18d (9223372036854775808)
 
                     Operand nLong = InstEmit.EmitSse2CvtDoubleToInt64OpF(context, nRes, false);
-                    Operand nLong2 = null;
+                    Operand nLong2 = default;
+
                     if (!signed)
                     {
                         nRes = context.AddIntrinsic(Intrinsic.X86Subpd, nRes, fpMaxValMask);

ARMeilleure/Instructions/InstEmitSimdHelper.cs

@@ -7,7 +7,7 @@ using System.Diagnostics;
 using System.Reflection;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -1279,9 +1279,11 @@ namespace ARMeilleure.Instructions
             context.MarkLabel(lblTrue);
         }
 
-        public static void EmitSseOrAvxExitFtzAndDazModesOpF(ArmEmitterContext context, Operand isTrue = null)
+        public static void EmitSseOrAvxExitFtzAndDazModesOpF(ArmEmitterContext context, Operand isTrue = default)
         {
-            isTrue ??= context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpcrFz)));
+            isTrue = isTrue == default 
+                ? context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpcrFz)))
+                : isTrue;
 
             Operand lblTrue = Label();
             context.BranchIfFalse(lblTrue, isTrue);
@@ -1455,7 +1457,7 @@ namespace ARMeilleure.Instructions
             }
             else
             {
-                Operand me = null;
+                Operand me = default;
 
                 if (byElem)
                 {
@@ -1625,7 +1627,7 @@ namespace ARMeilleure.Instructions
         {
             ThrowIfInvalid(index, size);
 
-            Operand res = null;
+            Operand res = default;
 
             switch (size)
             {

ARMeilleure/Instructions/InstEmitSimdHelper32.cs

@@ -7,7 +7,7 @@ using System.Reflection;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -1127,7 +1127,7 @@ namespace ARMeilleure.Instructions
         {
             ThrowIfInvalid(index, size);
 
-            Operand res = null;
+            Operand res = default;
 
             switch (size)
             {

ARMeilleure/Instructions/InstEmitSimdLogical.cs

@@ -6,7 +6,7 @@ using System.Diagnostics;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitSimdLogical32.cs

@@ -5,7 +5,7 @@ using ARMeilleure.Translation;
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper32;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitSimdMemory.cs

@@ -6,7 +6,7 @@ using System.Diagnostics;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitMemoryHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitSimdMemory32.cs

@@ -5,7 +5,7 @@ using ARMeilleure.Translation;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitMemoryHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitSimdMove.cs

@@ -6,7 +6,7 @@ using System.Reflection;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -639,7 +639,7 @@ namespace ARMeilleure.Instructions
 
             if (Optimizations.UseSsse3)
             {
-                Operand mask = null;
+                Operand mask = default;
 
                 if (op.Size < 3)
                 {
@@ -707,7 +707,7 @@ namespace ARMeilleure.Instructions
             {
                 if (op.RegisterSize == RegisterSize.Simd128)
                 {
-                    Operand mask = null;
+                    Operand mask = default;
 
                     if (op.Size < 3)
                     {

ARMeilleure/Instructions/InstEmitSimdMove32.cs

@@ -6,7 +6,7 @@ using System;
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper32;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -267,7 +267,7 @@ namespace ARMeilleure.Instructions
                     Operand selectedIndex = context.ZeroExtend8(OperandType.I32, context.VectorExtract8(m, index + op.Im));
 
                     Operand inRange = context.ICompareLess(selectedIndex, Const(byteLength));
-                    Operand elemRes = null; // Note: This is I64 for ease of calculation.
+                    Operand elemRes = default; // Note: This is I64 for ease of calculation.
 
                     // TODO: Branching rather than conditional select.
 
@@ -325,7 +325,7 @@ namespace ARMeilleure.Instructions
             {
                 EmitVectorShuffleOpSimd32(context, (m, d) =>
                 {
-                    Operand mask = null;
+                    Operand mask = default;
 
                     if (op.Size < 3)
                     {
@@ -467,7 +467,7 @@ namespace ARMeilleure.Instructions
                 {
                     if (op.RegisterSize == RegisterSize.Simd128)
                     {
-                        Operand mask = null;
+                        Operand mask = default;
 
                         if (op.Size < 3)
                         {

ARMeilleure/Instructions/InstEmitSimdShift.cs

@@ -9,7 +9,7 @@ using System.Reflection;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {
@@ -39,7 +39,7 @@ namespace ARMeilleure.Instructions
 
                 Operand dLow = context.VectorZeroUpper64(d);
 
-                Operand mask = null;
+                Operand mask = default;
 
                 switch (op.Size + 1)
                 {

ARMeilleure/Instructions/InstEmitSimdShift32.cs

@@ -8,7 +8,7 @@ using System.Reflection;
 using static ARMeilleure.Instructions.InstEmitHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper;
 using static ARMeilleure.Instructions.InstEmitSimdHelper32;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitSystem.cs

@@ -6,7 +6,7 @@ using System;
 using System.Reflection;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/Instructions/InstEmitSystem32.cs

@@ -6,7 +6,7 @@ using System;
 using System.Reflection;
 
 using static ARMeilleure.Instructions.InstEmitHelper;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Instructions
 {

ARMeilleure/IntermediateRepresentation/BasicBlock.cs

@@ -1,37 +1,47 @@
 using System;
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 
 namespace ARMeilleure.IntermediateRepresentation
 {
-    class BasicBlock : IIntrusiveListNode<BasicBlock>
+    class BasicBlock : IEquatable<BasicBlock>, IIntrusiveListNode<BasicBlock>
     {
-        private readonly List<BasicBlock> _successors;
+        private const uint MaxSuccessors = 2;
+
+        private int _succCount;
+        private BasicBlock _succ0;
+        private BasicBlock _succ1;
+        private HashSet<BasicBlock> _domFrontiers;
 
         public int Index { get; set; }
-
         public BasicBlockFrequency Frequency { get; set; }
-
         public BasicBlock ListPrevious { get; set; }
         public BasicBlock ListNext { get; set; }
-
+        public IntrusiveList<Operation> Operations { get; }
-        public IntrusiveList<Node> Operations { get; }
-
         public List<BasicBlock> Predecessors { get; }
-
-        public HashSet<BasicBlock> DominanceFrontiers { get; }
         public BasicBlock ImmediateDominator { get; set; }
 
-        public int SuccessorCount => _successors.Count;
+        public int SuccessorsCount => _succCount;
+
+        public HashSet<BasicBlock> DominanceFrontiers
+        {
+            get
+            {
+                if (_domFrontiers == null)
+                {
+                    _domFrontiers = new HashSet<BasicBlock>();
+                }
+
+                return _domFrontiers;
+            }
+        }
 
         public BasicBlock() : this(index: -1) { }
 
         public BasicBlock(int index)
         {
-            _successors = new List<BasicBlock>();
+            Operations = new IntrusiveList<Operation>();
-
-            Operations = new IntrusiveList<Node>();
             Predecessors = new List<BasicBlock>();
-            DominanceFrontiers = new HashSet<BasicBlock>();
 
             Index = index;
         }
@@ -40,54 +50,92 @@ namespace ARMeilleure.IntermediateRepresentation
         {
             if (block == null)
             {
-                throw new ArgumentNullException(nameof(block));
+                ThrowNull(nameof(block));
+            }
+
+            if ((uint)_succCount + 1 > MaxSuccessors)
+            {
+                ThrowSuccessorOverflow();
             }
 
             block.Predecessors.Add(this);
 
-            _successors.Add(block);
+            GetSuccessorUnsafe(_succCount++) = block;
         }
 
         public void RemoveSuccessor(int index)
         {
-            BasicBlock oldBlock = _successors[index];
+            if ((uint)index >= (uint)_succCount)
+            {
+                ThrowOutOfRange(nameof(index));
+            }
+
+            ref BasicBlock oldBlock = ref GetSuccessorUnsafe(index);
 
             oldBlock.Predecessors.Remove(this);
+            oldBlock = null;
 
-            _successors.RemoveAt(index);
+            if (index == 0)
+            {
+                _succ0 = _succ1;
+            }
+
+            _succCount--;
         }
 
         public BasicBlock GetSuccessor(int index)
         {
-            return _successors[index];
+            if ((uint)index >= (uint)_succCount)
+            {
+                ThrowOutOfRange(nameof(index));
+            }
+
+            return GetSuccessorUnsafe(index);
+        }
+
+        private ref BasicBlock GetSuccessorUnsafe(int index)
+        {
+            return ref Unsafe.Add(ref _succ0, index);
         }
 
         public void SetSuccessor(int index, BasicBlock block)
         {
             if (block == null)
             {
-                throw new ArgumentNullException(nameof(block));
+                ThrowNull(nameof(block));
             }
 
-            BasicBlock oldBlock = _successors[index];
+            if ((uint)index >= (uint)_succCount)
+            {
+                ThrowOutOfRange(nameof(index));
+            }
+
+            ref BasicBlock oldBlock = ref GetSuccessorUnsafe(index);
 
             oldBlock.Predecessors.Remove(this);
             block.Predecessors.Add(this);
             
-            _successors[index] = block;
+            oldBlock = block;
         }
 
-        public void Append(Node node)
+        public void Append(Operation node)
         {
-            var lastOp = Operations.Last as Operation;
+            Operation last = Operations.Last;
 
             // Append node before terminal or to end if no terminal.
-            switch (lastOp?.Instruction)
+            if (last == default)
+            {
+                Operations.AddLast(node);
+
+                return;
+            }
+
+            switch (last.Instruction)
             {
                 case Instruction.Return:
                 case Instruction.Tailcall:
                 case Instruction.BranchIf:
-                    Operations.AddBefore(lastOp, node);
+                    Operations.AddBefore(last, node);
                     break;
 
                 default:
@@ -96,9 +144,23 @@ namespace ARMeilleure.IntermediateRepresentation
             }
         }
 
-        public Node GetLastOp()
+        private static void ThrowNull(string name) => throw new ArgumentNullException(name);
+        private static void ThrowOutOfRange(string name) => throw new ArgumentOutOfRangeException(name);
+        private static void ThrowSuccessorOverflow() => throw new OverflowException($"BasicBlock can only have {MaxSuccessors} successors.");
+
+        public bool Equals(BasicBlock other)
         {
-            return Operations.Last;
+            return other == this;
+        }
+
+        public override bool Equals(object obj)
+        {
+            return Equals(obj as BasicBlock);
+        }
+
+        public override int GetHashCode()
+        {
+            return base.GetHashCode();
         }
     }
 }

ARMeilleure/IntermediateRepresentation/IIntrusiveListNode.cs

@@ -1,6 +1,6 @@
 namespace ARMeilleure.IntermediateRepresentation
 {
-    interface IIntrusiveListNode<T> where T : class
+    interface IIntrusiveListNode<T>
     {
         T ListPrevious { get; set; }
         T ListNext { get; set; }

ARMeilleure/IntermediateRepresentation/Instruction.cs

@@ -1,6 +1,6 @@
 namespace ARMeilleure.IntermediateRepresentation
 {
-    enum Instruction
+    enum Instruction : ushort
     {
         Add,
         BitwiseAnd,
@@ -63,6 +63,7 @@ namespace ARMeilleure.IntermediateRepresentation
         Extended,
         Fill,
         LoadFromContext,
+        Phi,
         Spill,
         SpillArg,
         StoreToContext

ARMeilleure/IntermediateRepresentation/Intrinsic.cs

@@ -1,6 +1,6 @@
 namespace ARMeilleure.IntermediateRepresentation
 {
-    enum Intrinsic
+    enum Intrinsic : ushort
     {
         X86Addpd,
         X86Addps,

ARMeilleure/IntermediateRepresentation/IntrinsicOperation.cs

@@ -1,12 +0,0 @@
-namespace ARMeilleure.IntermediateRepresentation
-{
-    class IntrinsicOperation : Operation
-    {
-        public Intrinsic Intrinsic { get; }
-
-        public IntrinsicOperation(Intrinsic intrin, Operand dest, params Operand[] sources) : base(Instruction.Extended, dest, sources)
-        {
-            Intrinsic = intrin;
-        }
-    }
-}

ARMeilleure/IntermediateRepresentation/IntrusiveList.cs

@@ -1,4 +1,6 @@
-using System.Diagnostics;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
 using System.Runtime.CompilerServices;
 
 namespace ARMeilleure.IntermediateRepresentation
@@ -7,7 +9,7 @@ namespace ARMeilleure.IntermediateRepresentation
     /// Represents a efficient linked list that stores the pointer on the object directly and does not allocate.
     /// </summary>
     /// <typeparam name="T">Type of the list items</typeparam>
-    class IntrusiveList<T> where T : class, IIntrusiveListNode<T>
+    class IntrusiveList<T> where T : IEquatable<T>, IIntrusiveListNode<T>
     {
         /// <summary>
         /// First item of the list, or null if empty.
@@ -24,22 +26,34 @@ namespace ARMeilleure.IntermediateRepresentation
         /// </summary>
         public int Count { get; private set; }
 
+        /// <summary>
+        /// Initializes a new instance of the <see cref="IntrusiveList{T}"/> class.
+        /// </summary>
+        /// <exception cref="ArgumentException"><typeparamref name="T"/> is not pointer sized.</exception>
+        public IntrusiveList()
+        {
+            if (Unsafe.SizeOf<T>() != IntPtr.Size)
+            {
+                throw new ArgumentException("T must be a reference type or a pointer sized struct.");
+            }
+        }
+
         /// <summary>
         /// Adds a item as the first item of the list.
         /// </summary>
         /// <param name="newNode">Item to be added</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void AddFirst(T newNode)
+        public T AddFirst(T newNode)
         {
-            if (First != null)
+            if (!EqualsNull(First))
             {
-                AddBefore(First, newNode);
+                return AddBefore(First, newNode);
             }
             else
             {
-                Debug.Assert(newNode.ListPrevious == null);
+                Debug.Assert(EqualsNull(newNode.ListPrevious));
-                Debug.Assert(newNode.ListNext == null);
+                Debug.Assert(EqualsNull(newNode.ListNext));
-                Debug.Assert(Last == null);
+                Debug.Assert(EqualsNull(Last));
 
                 First = newNode;
                 Last = newNode;
@@ -47,6 +61,8 @@ namespace ARMeilleure.IntermediateRepresentation
                 Debug.Assert(Count == 0);
 
                 Count = 1;
+
+                return newNode;
             }
         }
 
@@ -55,17 +71,17 @@ namespace ARMeilleure.IntermediateRepresentation
         /// </summary>
         /// <param name="newNode">Item to be added</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void AddLast(T newNode)
+        public T AddLast(T newNode)
         {
-            if (Last != null)
+            if (!EqualsNull(Last))
             {
-                AddAfter(Last, newNode);
+                return AddAfter(Last, newNode);
             }
             else
             {
-                Debug.Assert(newNode.ListPrevious == null);
+                Debug.Assert(EqualsNull(newNode.ListPrevious));
-                Debug.Assert(newNode.ListNext == null);
+                Debug.Assert(EqualsNull(newNode.ListNext));
-                Debug.Assert(First == null);
+                Debug.Assert(EqualsNull(First));
 
                 First = newNode;
                 Last = newNode;
@@ -73,6 +89,8 @@ namespace ARMeilleure.IntermediateRepresentation
                 Debug.Assert(Count == 0);
 
                 Count = 1;
+
+                return newNode;
             }
         }
 
@@ -85,20 +103,20 @@ namespace ARMeilleure.IntermediateRepresentation
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public T AddBefore(T node, T newNode)
         {
-            Debug.Assert(newNode.ListPrevious == null);
+            Debug.Assert(EqualsNull(newNode.ListPrevious));
-            Debug.Assert(newNode.ListNext == null);
+            Debug.Assert(EqualsNull(newNode.ListNext));
 
             newNode.ListPrevious = node.ListPrevious;
             newNode.ListNext = node;
 
             node.ListPrevious = newNode;
 
-            if (newNode.ListPrevious != null)
+            if (!EqualsNull(newNode.ListPrevious))
             {
                 newNode.ListPrevious.ListNext = newNode;
             }
 
-            if (First == node)
+            if (Equals(First, node))
             {
                 First = newNode;
             }
@@ -117,20 +135,20 @@ namespace ARMeilleure.IntermediateRepresentation
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public T AddAfter(T node, T newNode)
         {
-            Debug.Assert(newNode.ListPrevious == null);
+            Debug.Assert(EqualsNull(newNode.ListPrevious));
-            Debug.Assert(newNode.ListNext == null);
+            Debug.Assert(EqualsNull(newNode.ListNext));
 
             newNode.ListPrevious = node;
             newNode.ListNext = node.ListNext;
 
             node.ListNext = newNode;
 
-            if (newNode.ListNext != null)
+            if (!EqualsNull(newNode.ListNext))
             {
                 newNode.ListNext.ListPrevious = newNode;
             }
 
-            if (Last == node)
+            if (Equals(Last, node))
             {
                 Last = newNode;
             }
@@ -147,32 +165,44 @@ namespace ARMeilleure.IntermediateRepresentation
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void Remove(T node)
         {
-            if (node.ListPrevious != null)
+            if (!EqualsNull(node.ListPrevious))
             {
                 node.ListPrevious.ListNext = node.ListNext;
             }
             else
             {
-                Debug.Assert(First == node);
+                Debug.Assert(Equals(First, node));
 
                 First = node.ListNext;
             }
 
-            if (node.ListNext != null)
+            if (!EqualsNull(node.ListNext))
             {
                 node.ListNext.ListPrevious = node.ListPrevious;
             }
             else
             {
-                Debug.Assert(Last == node);
+                Debug.Assert(Equals(Last, node));
 
                 Last = node.ListPrevious;
             }
 
-            node.ListPrevious = null;
+            node.ListPrevious = default;
-            node.ListNext = null;
+            node.ListNext = default;
 
             Count--;
         }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static bool EqualsNull(T a)
+        {
+            return EqualityComparer<T>.Default.Equals(a, default);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static bool Equals(T a, T b)
+        {
+            return EqualityComparer<T>.Default.Equals(a, b);
+        }
     }
 }

ARMeilleure/IntermediateRepresentation/MemoryOperand.cs

@@ -1,29 +1,54 @@
+using System;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+
 namespace ARMeilleure.IntermediateRepresentation
 {
-    class MemoryOperand : Operand
+    unsafe struct MemoryOperand
     {
-        public Operand BaseAddress { get; set; }
+        private struct Data
-        public Operand Index       { get; set; }
-
-        public Multiplier Scale { get; private set; }
-
-        public int Displacement { get; private set; }
-
-        public MemoryOperand() { }
-
-        public MemoryOperand With(
-            OperandType type,
-            Operand     baseAddress,
-            Operand     index        = null,
-            Multiplier  scale        = Multiplier.x1,
-            int         displacement = 0)
         {
-            With(OperandKind.Memory, type);
+#pragma warning disable CS0649
-            BaseAddress  = baseAddress;
+            public byte Kind;
-            Index        = index;
+            public byte Type;
-            Scale        = scale;
+#pragma warning restore CS0649
-            Displacement = displacement;
+            public byte Scale;
-            return this;
+            public Operand BaseAddress;
+            public Operand Index;
+            public int Displacement;
+        }
+
+        private Data* _data;
+
+        public MemoryOperand(Operand operand)
+        {
+            Debug.Assert(operand.Kind == OperandKind.Memory);
+
+            _data = (Data*)Unsafe.As<Operand, IntPtr>(ref operand);
+        }
+
+        public Operand BaseAddress
+        {
+            get => _data->BaseAddress;
+            set => _data->BaseAddress = value; 
+        }
+
+        public Operand Index
+        {
+            get => _data->Index;
+            set => _data->Index = value; 
+        }
+
+        public Multiplier Scale
+        {
+            get => (Multiplier)_data->Scale;
+            set => _data->Scale = (byte)value;
+        }
+
+        public int Displacement
+        {
+            get => _data->Displacement;
+            set => _data->Displacement = value;
         }
     }
 }

ARMeilleure/IntermediateRepresentation/Node.cs

@@ -1,309 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-namespace ARMeilleure.IntermediateRepresentation
-{
-    class Node : IIntrusiveListNode<Node>
-    {
-        public Node ListPrevious { get; set; }
-        public Node ListNext { get; set; }
-
-        public Operand Destination
-        {
-            get => _destinations.Count != 0 ? GetDestination(0) : null;
-            set => SetDestination(value);
-        }
-
-        private readonly List<Operand> _destinations;
-        private readonly List<Operand> _sources;
-        private bool _clearedDest;
-
-        public int DestinationsCount => _destinations.Count;
-        public int SourcesCount      => _sources.Count;
-
-        private void Resize(List<Operand> list, int size)
-        {
-            if (list.Count > size)
-            {
-                list.RemoveRange(size, list.Count - size);
-            } 
-            else
-            {
-                while (list.Count < size)
-                {
-                    list.Add(null);
-                }
-            }
-        }
-
-        public Node()
-        {
-            _destinations = new List<Operand>();
-            _sources = new List<Operand>();
-        }
-
-        public Node(Operand destination, int sourcesCount) : this()
-        {
-            Destination = destination;
-
-            Resize(_sources, sourcesCount);
-        }
-
-        private void Reset(int sourcesCount)
-        {
-            _clearedDest = true;
-            _sources.Clear();
-            ListPrevious = null;
-            ListNext = null;
-
-            Resize(_sources, sourcesCount);
-        }
-
-        public Node With(Operand destination, int sourcesCount)
-        {
-            Reset(sourcesCount);
-            Destination = destination;
-
-            return this;
-        }
-
-        public Node With(Operand[] destinations, int sourcesCount)
-        {
-            Reset(sourcesCount);
-            SetDestinations(destinations ?? throw new ArgumentNullException(nameof(destinations)));
-
-            return this;
-        }
-
-        public Operand GetDestination(int index)
-        {
-            return _destinations[index];
-        }
-
-        public Operand GetSource(int index)
-        {
-            return _sources[index];
-        }
-
-        public void SetDestination(int index, Operand destination)
-        {
-            if (!_clearedDest) 
-            {
-                RemoveAssignment(_destinations[index]);
-            }
-
-            AddAssignment(destination);
-
-            _clearedDest = false;
-
-            _destinations[index] = destination;
-        }
-
-        public void SetSource(int index, Operand source)
-        {
-            RemoveUse(_sources[index]);
-
-            AddUse(source);
-
-            _sources[index] = source;
-        }
-
-        private void RemoveOldDestinations()
-        {
-            if (!_clearedDest)
-            {
-                for (int index = 0; index < _destinations.Count; index++)
-                {
-                    RemoveAssignment(_destinations[index]);
-                }
-            }
-
-            _clearedDest = false;
-        }
-
-        public void SetDestination(Operand destination)
-        {
-            RemoveOldDestinations();
-
-            if (destination == null)
-            {
-                _destinations.Clear();
-                _clearedDest = true;
-            }
-            else
-            {
-                Resize(_destinations, 1);
-
-                _destinations[0] = destination;
-
-                AddAssignment(destination);
-            }
-        }
-
-        public void SetDestinations(Operand[] destinations)
-        {
-            RemoveOldDestinations();
-
-            Resize(_destinations, destinations.Length);
-
-            for (int index = 0; index < destinations.Length; index++)
-            {
-                Operand newOp = destinations[index];
-
-                _destinations[index] = newOp;
-
-                AddAssignment(newOp);
-            }
-        }
-
-        private void RemoveOldSources()
-        {
-            for (int index = 0; index < _sources.Count; index++)
-            {
-                RemoveUse(_sources[index]);
-            }
-        }
-
-        public void SetSource(Operand source)
-        {
-            RemoveOldSources();
-
-            if (source == null)
-            {
-                _sources.Clear();
-            }
-            else
-            {
-                Resize(_sources, 1);
-
-                _sources[0] = source;
-
-                AddUse(source);
-            }
-        }
-
-        public void SetSources(Operand[] sources)
-        {
-            RemoveOldSources();
-
-            Resize(_sources, sources.Length);
-
-            for (int index = 0; index < sources.Length; index++)
-            {
-                Operand newOp = sources[index];
-
-                _sources[index] = newOp;
-
-                AddUse(newOp);
-            }
-        }
-
-        private void AddAssignment(Operand op)
-        {
-            if (op == null)
-            {
-                return;
-            }
-
-            if (op.Kind == OperandKind.LocalVariable)
-            {
-                op.Assignments.Add(this);
-            }
-            else if (op.Kind == OperandKind.Memory)
-            {
-                MemoryOperand memOp = (MemoryOperand)op;
-
-                if (memOp.BaseAddress != null)
-                {
-                    memOp.BaseAddress.Assignments.Add(this);
-                }
-                
-                if (memOp.Index != null)
-                {
-                    memOp.Index.Assignments.Add(this);
-                }
-            }
-        }
-
-        private void RemoveAssignment(Operand op)
-        {
-            if (op == null)
-            {
-                return;
-            }
-
-            if (op.Kind == OperandKind.LocalVariable)
-            {
-                op.Assignments.Remove(this);
-            }
-            else if (op.Kind == OperandKind.Memory)
-            {
-                MemoryOperand memOp = (MemoryOperand)op;
-
-                if (memOp.BaseAddress != null)
-                {
-                    memOp.BaseAddress.Assignments.Remove(this);
-                }
-
-                if (memOp.Index != null)
-                {
-                    memOp.Index.Assignments.Remove(this);
-                }
-            }
-        }
-
-        private void AddUse(Operand op)
-        {
-            if (op == null)
-            {
-                return;
-            }
-
-            if (op.Kind == OperandKind.LocalVariable)
-            {
-                op.Uses.Add(this);
-            }
-            else if (op.Kind == OperandKind.Memory)
-            {
-                MemoryOperand memOp = (MemoryOperand)op;
-
-                if (memOp.BaseAddress != null)
-                {
-                    memOp.BaseAddress.Uses.Add(this);
-                }
-
-                if (memOp.Index != null)
-                {
-                    memOp.Index.Uses.Add(this);
-                }
-            }
-        }
-
-        private void RemoveUse(Operand op)
-        {
-            if (op == null)
-            {
-                return;
-            }
-
-            if (op.Kind == OperandKind.LocalVariable)
-            {
-                op.Uses.Remove(this);
-            }
-            else if (op.Kind == OperandKind.Memory)
-            {
-                MemoryOperand memOp = (MemoryOperand)op;
-
-                if (memOp.BaseAddress != null)
-                {
-                    memOp.BaseAddress.Uses.Remove(this);
-                }
-
-                if (memOp.Index != null)
-                {
-                    memOp.Index.Uses.Remove(this);
-                }
-            }
-        }
-    }
-}

ARMeilleure/IntermediateRepresentation/Operand.cs

@@ -1,3 +1,4 @@
+using ARMeilleure.Common;
 using ARMeilleure.Translation.PTC;
 using System;
 using System.Collections.Generic;
@@ -6,96 +7,108 @@ using System.Runtime.CompilerServices;
 
 namespace ARMeilleure.IntermediateRepresentation
 {
-    class Operand
+    unsafe struct Operand : IEquatable<Operand>
     {
-        public OperandKind Kind { get; private set; }
+        internal struct Data
-        public OperandType Type { get; private set; }
+        {
+            public byte Kind;
+            public byte Type;
+            public byte SymbolType;
+            public ushort AssignmentsCount;
+            public ushort AssignmentsCapacity;
+            public ushort UsesCount;
+            public ushort UsesCapacity;
+            public Operation* Assignments;
+            public Operation* Uses;
+            public ulong Value;
+            public ulong SymbolValue;
+        }
 
-        public ulong Value { get; private set; }
+        private Data* _data;
 
-        public List<Node> Assignments { get; }
+        public OperandKind Kind
-        public List<Node> Uses        { get; }
+        {
+            get => (OperandKind)_data->Kind;
+            private set => _data->Kind = (byte)value;
+        }
+
+        public OperandType Type
+        {
+            get => (OperandType)_data->Type;
+            private set => _data->Type = (byte)value;
+        }
+
+        public ulong Value
+        {
+            get => _data->Value;
+            private set => _data->Value = value;
+        }
+
+        public Symbol Symbol
+        {
+            get
+            {
+                Debug.Assert(Kind != OperandKind.Memory);
+
+                return new Symbol((SymbolType)_data->SymbolType, _data->SymbolValue);
+            }
+            private set
+            {
+                Debug.Assert(Kind != OperandKind.Memory);
+
+                if (value.Type == SymbolType.None)
+                {
+                    _data->SymbolType = (byte)SymbolType.None;
+                }
+                else
+                {
+                    _data->SymbolType = (byte)value.Type;
+                    _data->SymbolValue = value.Value;
+                }
+            }
+        }
+
+        public ReadOnlySpan<Operation> Assignments
+        {
+            get
+            {
+                Debug.Assert(Kind != OperandKind.Memory);
+
+                return new ReadOnlySpan<Operation>(_data->Assignments, _data->AssignmentsCount);
+            }
+        }
+
+        public ReadOnlySpan<Operation> Uses
+        {
+            get
+            {
+                Debug.Assert(Kind != OperandKind.Memory);
+
+                return new ReadOnlySpan<Operation>(_data->Uses, _data->UsesCount);
+            }
+        }
+
+        public int UsesCount => _data->UsesCount;
+        public int AssignmentsCount => _data->AssignmentsCount;
 
-        public Symbol Symbol { get; private set; }
         public bool Relocatable => Symbol.Type != SymbolType.None;
 
-        public Operand()
-        {
-            Assignments = new List<Node>();
-            Uses        = new List<Node>();
-        }
-
-        public Operand(OperandKind kind, OperandType type = OperandType.None) : this()
-        {
-            Kind = kind;
-            Type = type;
-        }
-
-        public Operand With(
-            OperandKind kind,
-            OperandType type = OperandType.None,
-            ulong value = 0,
-            Symbol symbol = default)
-        {
-            Kind = kind;
-            Type = type;
-
-            Value = value;
-
-            Symbol = symbol;
-
-            Assignments.Clear();
-            Uses.Clear();
-
-            return this;
-        }
-
-        public Operand With(int value)
-        {
-            return With(OperandKind.Constant, OperandType.I32, (uint)value);
-        }
-
-        public Operand With(uint value)
-        {
-            return With(OperandKind.Constant, OperandType.I32, value);
-        }
-
-        public Operand With(long value)
-        {
-            return With(OperandKind.Constant, OperandType.I64, (ulong)value);
-        }
-
-        public Operand With(long value, Symbol symbol)
-        {
-            return With(OperandKind.Constant, OperandType.I64, (ulong)value, symbol);
-        }
-
-        public Operand With(ulong value)
-        {
-            return With(OperandKind.Constant, OperandType.I64, value);
-        }
-
-        public Operand With(float value)
-        {
-            return With(OperandKind.Constant, OperandType.FP32, (ulong)BitConverter.SingleToInt32Bits(value));
-        }
-
-        public Operand With(double value)
-        {
-            return With(OperandKind.Constant, OperandType.FP64, (ulong)BitConverter.DoubleToInt64Bits(value));
-        }
-
-        public Operand With(int index, RegisterType regType, OperandType type)
-        {
-            return With(OperandKind.Register, type, (ulong)((int)regType << 24 | index));
-        }
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public Register GetRegister()
         {
+            Debug.Assert(Kind == OperandKind.Register);
+
             return new Register((int)Value & 0xffffff, (RegisterType)(Value >> 24));
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public MemoryOperand GetMemory()
+        {
+            Debug.Assert(Kind == OperandKind.Memory);
+
+            return new MemoryOperand(this);
+        }
+
         public int GetLocalNumber()
         {
             Debug.Assert(Kind == OperandKind.LocalVariable);
@@ -133,6 +146,11 @@ namespace ARMeilleure.IntermediateRepresentation
             return BitConverter.Int64BitsToDouble((long)Value);
         }
 
+        internal ref ulong GetValueUnsafe()
+        {
+            return ref _data->Value;
+        }
+
         internal void NumberLocal(int number)
         {
             if (Kind != OperandKind.LocalVariable)
@@ -143,6 +161,158 @@ namespace ARMeilleure.IntermediateRepresentation
             Value = (ulong)number;
         }
 
+        public void AddAssignment(Operation operation)
+        {
+            if (Kind == OperandKind.LocalVariable)
+            {
+                Add(operation, ref _data->Assignments, ref _data->AssignmentsCount, ref _data->AssignmentsCapacity);
+            }
+            else if (Kind == OperandKind.Memory)
+            {
+                MemoryOperand memOp = GetMemory();
+                Operand addr = memOp.BaseAddress;
+                Operand index = memOp.Index;
+
+                if (addr != default)
+                {
+                    Add(operation, ref addr._data->Assignments, ref addr._data->AssignmentsCount, ref addr._data->AssignmentsCapacity);
+                }
+                
+                if (index != default)
+                {
+                    Add(operation, ref index._data->Assignments, ref index._data->AssignmentsCount, ref index._data->AssignmentsCapacity);
+                }
+            }
+        }
+
+        public void RemoveAssignment(Operation operation)
+        {
+            if (Kind == OperandKind.LocalVariable)
+            {
+                Remove(operation, ref _data->Assignments, ref _data->AssignmentsCount);
+            }
+            else if (Kind == OperandKind.Memory)
+            {
+                MemoryOperand memOp = GetMemory();
+                Operand addr = memOp.BaseAddress;
+                Operand index = memOp.Index;
+
+                if (addr != default)
+                {
+                    Remove(operation, ref addr._data->Assignments, ref addr._data->AssignmentsCount);
+                }
+
+                if (index != default)
+                {
+                    Remove(operation, ref index._data->Assignments, ref index._data->AssignmentsCount);
+                }
+            }
+        }
+
+        public void AddUse(Operation operation)
+        {
+            if (Kind == OperandKind.LocalVariable)
+            {
+                Add(operation, ref _data->Uses, ref _data->UsesCount, ref _data->UsesCapacity);
+            }
+            else if (Kind == OperandKind.Memory)
+            {
+                MemoryOperand memOp = GetMemory();
+                Operand addr = memOp.BaseAddress;
+                Operand index = memOp.Index;
+
+                if (addr != default)
+                {
+                    Add(operation, ref addr._data->Uses, ref addr._data->UsesCount, ref addr._data->UsesCapacity);
+                }
+
+                if (index != default)
+                {
+                    Add(operation, ref index._data->Uses, ref index._data->UsesCount, ref index._data->UsesCapacity);
+                }
+            }
+        }
+
+        public void RemoveUse(Operation operation)
+        {
+            if (Kind == OperandKind.LocalVariable)
+            {
+                Remove(operation, ref _data->Uses, ref _data->UsesCount);
+            }
+            else if (Kind == OperandKind.Memory)
+            {
+                MemoryOperand memOp = GetMemory();
+                Operand addr = memOp.BaseAddress;
+                Operand index = memOp.Index;
+
+                if (addr != default)
+                {
+                    Remove(operation, ref addr._data->Uses, ref addr._data->UsesCount);
+                }
+
+                if (index != default)
+                {
+                    Remove(operation, ref index._data->Uses, ref index._data->UsesCount);
+                }
+            }
+        }
+
+        private static void New<T>(ref T* data, ref ushort count, ref ushort capacity, ushort initialCapacity) where T : unmanaged
+        {
+            count = 0;
+            capacity = initialCapacity;
+            data = Allocators.References.Allocate<T>(initialCapacity);
+        }
+
+        private static void Add<T>(T item, ref T* data, ref ushort count, ref ushort capacity) where T : unmanaged
+        {
+            if (count < capacity)
+            {
+                data[(uint)count++] = item;
+
+                return;
+            }
+
+            // Could not add item in the fast path, fallback onto the slow path.
+            ExpandAdd(item, ref data, ref count, ref capacity);
+
+            static void ExpandAdd(T item, ref T* data, ref ushort count, ref ushort capacity)
+            {
+                ushort newCount = checked((ushort)(count + 1));
+                ushort newCapacity = (ushort)Math.Min(capacity * 2, ushort.MaxValue);
+
+                var oldSpan = new Span<T>(data, count);
+
+                capacity = newCapacity;
+                data = Allocators.References.Allocate<T>(capacity);
+
+                oldSpan.CopyTo(new Span<T>(data, count));
+
+                data[count] = item;
+                count = newCount;
+            }
+        }
+
+        private static void Remove<T>(in T item, ref T* data, ref ushort count) where T : unmanaged
+        {
+            var span = new Span<T>(data, count);
+
+            for (int i = 0; i < span.Length; i++)
+            {
+                if (EqualityComparer<T>.Default.Equals(span[i], item))
+                {
+                    if (i + 1 < count)
+                    {
+                        span.Slice(i + 1).CopyTo(span.Slice(i));
+                    }
+
+                    count--;
+
+                    return;
+                }
+            }
+        }
+
         public override int GetHashCode()
         {
             if (Kind == OperandKind.LocalVariable)
@@ -154,5 +324,201 @@ namespace ARMeilleure.IntermediateRepresentation
                 return (int)Value ^ ((int)Kind << 16) ^ ((int)Type << 20);
             }
         }
+
+        public bool Equals(Operand operand)
+        {
+            return operand._data == _data;
+        }
+
+        public override bool Equals(object obj)
+        {
+            return obj is Operand operand && Equals(operand);
+        }
+
+        public static bool operator ==(Operand a, Operand b)
+        {
+            return a.Equals(b);
+        }
+
+        public static bool operator !=(Operand a, Operand b)
+        {
+            return !a.Equals(b);
+        }
+
+        public static class Factory
+        {
+            private const int InternTableSize = 256;
+            private const int InternTableProbeLength = 8;
+
+            [ThreadStatic]
+            private static Data* _internTable;
+
+            private static Data* InternTable
+            {
+                get
+                {
+                    if (_internTable == null)
+                    {
+                        _internTable = (Data*)NativeAllocator.Instance.Allocate((uint)sizeof(Data) * InternTableSize);
+
+                        // Make sure the table is zeroed.
+                        new Span<Data>(_internTable, InternTableSize).Clear();
+                    }
+
+                    return _internTable;
+                }
+            }
+
+            private static Operand Make(OperandKind kind, OperandType type, ulong value, Symbol symbol = default)
+            {
+                Debug.Assert(kind != OperandKind.None);
+
+                Data* data = null;
+
+                // If constant or register, then try to look up in the intern table before allocating.
+                if (kind == OperandKind.Constant || kind == OperandKind.Register)
+                {
+                    uint hash = (uint)HashCode.Combine(kind, type, value);
+
+                    // Look in the next InternTableProbeLength slots for a match.
+                    for (uint i = 0; i < InternTableProbeLength; i++)
+                    {
+                        Operand interned = new();
+                        interned._data = &InternTable[(hash + i) % InternTableSize];
+
+                        // If slot matches the allocation request then return that slot.
+                        if (interned.Kind == kind && interned.Type == type && interned.Value == value && interned.Symbol == symbol)
+                        {
+                            return interned;
+                        }
+                        // Otherwise if the slot is not occupied, we store in that slot.
+                        else if (interned.Kind == OperandKind.None)
+                        {
+                            data = interned._data;
+
+                            break;
+                        }
+                    }
+                }
+
+                // If we could not get a slot from the intern table, we allocate somewhere else and store there.
+                if (data == null)
+                {
+                    data = Allocators.Operands.Allocate<Data>();
+                }
+
+                *data = default;
+
+                Operand result = new();
+                result._data = data;
+                result.Value = value;
+                result.Kind = kind;
+                result.Type = type;
+
+                if (kind != OperandKind.Memory)
+                {
+                    result.Symbol = symbol;
+                }
+
+                // If local variable, then the use and def list is initialized with default sizes.
+                if (kind == OperandKind.LocalVariable)
+                {
+                    New(ref result._data->Assignments, ref result._data->AssignmentsCount, ref result._data->AssignmentsCapacity, 1);
+                    New(ref result._data->Uses, ref result._data->UsesCount, ref result._data->UsesCapacity, 4);
+                }
+
+                return result;
+            }
+
+            public static Operand Const(OperandType type, long value)
+            {
+                Debug.Assert(type is OperandType.I32 or OperandType.I64);
+
+                return type == OperandType.I32 ? Const((int)value) : Const(value);
+            }
+
+            public static Operand Const(bool value)
+            {
+                return Const(value ? 1 : 0);
+            }
+
+            public static Operand Const(int value)
+            {
+                return Const((uint)value);
+            }
+
+            public static Operand Const(uint value)
+            {
+                return Make(OperandKind.Constant, OperandType.I32, value);
+            }
+
+            public static Operand Const(long value)
+            {
+                return Const(value, symbol: default);
+            }
+
+            public static Operand Const<T>(ref T reference, Symbol symbol = default)
+            {
+                return Const((long)Unsafe.AsPointer(ref reference), symbol);
+            }
+
+            public static Operand Const(long value, Symbol symbol)
+            {
+                return Make(OperandKind.Constant, OperandType.I64, (ulong)value, symbol);
+            }
+
+            public static Operand Const(ulong value)
+            {
+                return Make(OperandKind.Constant, OperandType.I64, value);
+            }
+
+            public static Operand ConstF(float value)
+            {
+                return Make(OperandKind.Constant, OperandType.FP32, (ulong)BitConverter.SingleToInt32Bits(value));
+            }
+
+            public static Operand ConstF(double value)
+            {
+                return Make(OperandKind.Constant, OperandType.FP64, (ulong)BitConverter.DoubleToInt64Bits(value));
+            }
+
+            public static Operand Label()
+            {
+                return Make(OperandKind.Label, OperandType.None, 0);
+            }
+
+            public static Operand Local(OperandType type)
+            {
+                return Make(OperandKind.LocalVariable, type, 0);
+            }
+
+            public static Operand Register(int index, RegisterType regType, OperandType type)
+            {
+                return Make(OperandKind.Register, type, (ulong)((int)regType << 24 | index));
+            }
+
+            public static Operand Undef()
+            {
+                return Make(OperandKind.Undefined, OperandType.None, 0);
+            }
+
+            public static Operand MemoryOp(
+                OperandType type,
+                Operand baseAddress,
+                Operand index = default,
+                Multiplier scale = Multiplier.x1,
+                int displacement = 0)
+            {
+                Operand result = Make(OperandKind.Memory, type, 0);
+
+                MemoryOperand memory = result.GetMemory();
+                memory.BaseAddress = baseAddress;
+                memory.Index = index;
+                memory.Scale = scale;
+                memory.Displacement = displacement;
+
+                return result;
+            }
+        }
     }
 }

ARMeilleure/IntermediateRepresentation/OperandHelper.cs

@@ -1,119 +0,0 @@
-using ARMeilleure.Common;
-using ARMeilleure.Translation.PTC;
-using System.Runtime.CompilerServices;
-
-namespace ARMeilleure.IntermediateRepresentation
-{
-    static class OperandHelper
-    {
-        public static Operand Const(OperandType type, long value)
-        {
-            return type == OperandType.I32 ? Operand().With((int)value) : Operand().With(value);
-        }
-
-        public static Operand Const(bool value)
-        {
-            return Operand().With(value ? 1 : 0);
-        }
-
-        public static Operand Const(int value)
-        {
-            return Operand().With(value);
-        }
-
-        public static Operand Const(uint value)
-        {
-            return Operand().With(value);
-        }
-
-        public static Operand Const(long value)
-        {
-            return Operand().With(value);
-        }
-
-        public static Operand Const(long value, Symbol symbol)
-        {
-            return Operand().With(value, symbol);
-        }
-
-        public static Operand Const(ulong value)
-        {
-            return Operand().With(value);
-        }
-
-        public static unsafe Operand Const<T>(ref T reference, Symbol symbol = default)
-        {
-            return Operand().With((long)Unsafe.AsPointer(ref reference), symbol);
-        }
-
-        public static Operand ConstF(float value)
-        {
-            return Operand().With(value);
-        }
-
-        public static Operand ConstF(double value)
-        {
-            return Operand().With(value);
-        }
-
-        public static Operand Label()
-        {
-            return Operand().With(OperandKind.Label);
-        }
-
-        public static Operand Local(OperandType type)
-        {
-            return Operand().With(OperandKind.LocalVariable, type);
-        }
-
-        public static Operand Register(int index, RegisterType regType, OperandType type)
-        {
-            return Operand().With(index, regType, type);
-        }
-
-        public static Operand Undef()
-        {
-            return Operand().With(OperandKind.Undefined);
-        }
-
-        public static MemoryOperand MemoryOp(
-            OperandType type,
-            Operand baseAddress,
-            Operand index = null,
-            Multiplier scale = Multiplier.x1,
-            int displacement = 0)
-        {
-            return MemoryOperand().With(type, baseAddress, index, scale, displacement);
-        }
-
-        #region "ThreadStaticPool"
-        public static void PrepareOperandPool(int groupId = 0)
-        {
-            ThreadStaticPool<Operand>.PreparePool(groupId, ChunkSizeLimit.Large);
-            ThreadStaticPool<MemoryOperand>.PreparePool(groupId, ChunkSizeLimit.Small);
-        }
-
-        private static Operand Operand()
-        {
-            return ThreadStaticPool<Operand>.Instance.Allocate();
-        }
-
-        private static MemoryOperand MemoryOperand()
-        {
-            return ThreadStaticPool<MemoryOperand>.Instance.Allocate();
-        }
-
-        public static void ResetOperandPool(int groupId = 0)
-        {
-            ThreadStaticPool<MemoryOperand>.ResetPool(groupId);
-            ThreadStaticPool<Operand>.ResetPool(groupId);
-        }
-
-        public static void DisposeOperandPools()
-        {
-            ThreadStaticPool<Operand>.DisposePools();
-            ThreadStaticPool<MemoryOperand>.DisposePools();
-        }
-        #endregion
-    }
-}

ARMeilleure/IntermediateRepresentation/OperandKind.cs

@@ -2,6 +2,7 @@ namespace ARMeilleure.IntermediateRepresentation
 {
     enum OperandKind
     {
+        None,
         Constant,
         Label,
         LocalVariable,

ARMeilleure/IntermediateRepresentation/Operation.cs

@@ -1,89 +1,180 @@
+using System;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+
 namespace ARMeilleure.IntermediateRepresentation
 {
-    class Operation : Node
+    unsafe struct Operation : IEquatable<Operation>, IIntrusiveListNode<Operation>
     {
-        public Instruction Instruction { get; private set; }
+        internal struct Data
-
-        public Operation() : base() { }
-
-        public Operation(
-            Instruction instruction,
-            Operand destination,
-            Operand[] sources) : base(destination, sources.Length)
         {
-            Instruction = instruction;
+            public ushort Instruction;
+            public ushort Intrinsic;
+            public ushort SourcesCount;
+            public ushort DestinationsCount;
+            public Operation ListPrevious;
+            public Operation ListNext;
+            public Operand* Destinations;
+            public Operand* Sources;
+        }
 
-            for (int index = 0; index < sources.Length; index++)
+        private Data* _data;
+
+        public Instruction Instruction
         {
-                SetSource(index, sources[index]);
+            get => (Instruction)_data->Instruction;
+            private set => _data->Instruction = (ushort)value;
+        }
+
+        public Intrinsic Intrinsic
+        {
+            get => (Intrinsic)_data->Intrinsic;
+            private set => _data->Intrinsic = (ushort)value;
+        }
+
+        public Operation ListPrevious
+        {
+            get => _data->ListPrevious;
+            set => _data->ListPrevious = value;
+        }
+
+        public Operation ListNext
+        {
+            get => _data->ListNext;
+            set => _data->ListNext = value;
+        }
+
+        public Operand Destination
+        {
+            get => _data->DestinationsCount != 0 ? GetDestination(0) : default;
+            set => SetDestination(value);
+        }
+
+        public int DestinationsCount => _data->DestinationsCount;
+        public int SourcesCount => _data->SourcesCount;
+
+        private Span<Operand> Destinations => new(_data->Destinations, _data->DestinationsCount);
+        private Span<Operand> Sources => new(_data->Sources, _data->SourcesCount);
+
+        public PhiOperation AsPhi()
+        {
+            Debug.Assert(Instruction == Instruction.Phi);
+
+            return new PhiOperation(this);
+        }
+
+        public Operand GetDestination(int index)
+        {
+            return Destinations[index];
+        }
+
+        public Operand GetSource(int index)
+        {
+            return Sources[index];
+        }
+
+        public void SetDestination(int index, Operand dest)
+        {
+            ref Operand curDest = ref Destinations[index];
+
+            RemoveAssignment(curDest);
+            AddAssignment(dest);
+
+            curDest = dest;
+        }
+
+        public void SetSource(int index, Operand src)
+        {
+            ref Operand curSrc = ref Sources[index];
+
+            RemoveUse(curSrc);
+            AddUse(src);
+
+            curSrc = src;
+        }
+
+        private void RemoveOldDestinations()
+        {
+            for (int i = 0; i < _data->DestinationsCount; i++)
+            {
+                RemoveAssignment(_data->Destinations[i]);
             }
         }
 
-        public Operation With(Instruction instruction, Operand destination)
+        public void SetDestination(Operand dest)
         {
-            With(destination, 0);
+            RemoveOldDestinations();
-            Instruction = instruction;
+
-            return this;
+            if (dest == default)
+            {
+                _data->DestinationsCount = 0;
+            }
+            else
+            {
+                EnsureCapacity(ref _data->Destinations, ref _data->DestinationsCount, 1);
+
+                _data->Destinations[0] = dest;
+
+                AddAssignment(dest);
+            }
         }
 
-        public Operation With(Instruction instruction, Operand destination, Operand[] sources)
+        public void SetDestinations(Operand[] dests)
         {
-            With(destination, sources.Length);
+            RemoveOldDestinations();
-            Instruction = instruction;
 
-            for (int index = 0; index < sources.Length; index++)
+            EnsureCapacity(ref _data->Destinations, ref _data->DestinationsCount, dests.Length);
+
+            for (int index = 0; index < dests.Length; index++)
             {
-                SetSource(index, sources[index]);
+                Operand newOp = dests[index];
+
+                _data->Destinations[index] = newOp;
+
+                AddAssignment(newOp);
             }
-            return this;
         }
 
-        public Operation With(Instruction instruction, Operand destination, 
+        private void RemoveOldSources()
-            Operand source0)
         {
-            With(destination, 1);
+            for (int index = 0; index < _data->SourcesCount; index++)
-            Instruction = instruction;
+            {
-
+                RemoveUse(_data->Sources[index]);
-            SetSource(0, source0);
+            }
-            return this;
         }
 
-        public Operation With(Instruction instruction, Operand destination,
+        public void SetSource(Operand src)
-            Operand source0, Operand source1)
         {
-            With(destination, 2);
+            RemoveOldSources();
-            Instruction = instruction;
 
-            SetSource(0, source0);
+            if (src == default)
-            SetSource(1, source1);
+            {
-            return this;
+                _data->SourcesCount = 0;
+            }
+            else
+            {
+                EnsureCapacity(ref _data->Sources, ref _data->SourcesCount, 1);
+
+                _data->Sources[0] = src;
+
+                AddUse(src);
+            }
         }
 
-        public Operation With(Instruction instruction, Operand destination, 
+        public void SetSources(Operand[] srcs)
-            Operand source0, Operand source1, Operand source2)
         {
-            With(destination, 3);
+            RemoveOldSources();
-            Instruction = instruction;
 
-            SetSource(0, source0);
+            EnsureCapacity(ref _data->Sources, ref _data->SourcesCount, srcs.Length);
-            SetSource(1, source1);
+
-            SetSource(2, source2);
+            for (int index = 0; index < srcs.Length; index++)
-            return this;
+            {
+                Operand newOp = srcs[index];
+
+                _data->Sources[index] = newOp;
+
+                AddUse(newOp);
             }
-
-        public Operation With(
-            Instruction instruction,
-            Operand[] destinations,
-            Operand[] sources)
-        {
-            With(destinations, sources.Length);
-            Instruction = instruction;
-
-            for (int index = 0; index < sources.Length; index++)
-            {
-                SetSource(index, sources[index]);
-            }
-            return this;
         }
 
         public void TurnIntoCopy(Operand source)
@@ -92,5 +183,194 @@ namespace ARMeilleure.IntermediateRepresentation
 
             SetSource(source);
         }
+
+        private void AddAssignment(Operand op)
+        {
+            if (op != default)
+            {
+                op.AddAssignment(this);
+            }
+        }
+
+        private void RemoveAssignment(Operand op)
+        {
+            if (op != default)
+            {
+                op.RemoveAssignment(this);
+            }
+        }
+
+        private void AddUse(Operand op)
+        {
+            if (op != default)
+            {
+                op.AddUse(this);
+            }
+        }
+
+        private void RemoveUse(Operand op)
+        {
+            if (op != default)
+            {
+                op.RemoveUse(this);
+            }
+        }
+
+        public bool Equals(Operation operation)
+        {
+            return operation._data == _data;
+        }
+
+        public override bool Equals(object obj)
+        {
+            return obj is Operation operation && Equals(operation);
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine((IntPtr)_data);
+        }
+
+        public static bool operator ==(Operation a, Operation b)
+        {
+            return a.Equals(b);
+        }
+
+        public static bool operator !=(Operation a, Operation b)
+        {
+            return !a.Equals(b);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static void EnsureCapacity(ref Operand* list, ref ushort capacity, int newCapacity)
+        {
+            if (newCapacity > ushort.MaxValue)
+            {
+                ThrowOverflow(newCapacity);
+            }
+            // We only need to allocate a new buffer if we're increasing the size.
+            else if (newCapacity > capacity)
+            {
+                list = Allocators.References.Allocate<Operand>((uint)newCapacity);
+            }
+
+            capacity = (ushort)newCapacity;
+        }
+
+        private static void ThrowOverflow(int count) =>
+            throw new OverflowException($"Exceeded maximum size for Source or Destinations. Required {count}.");
+
+        public static class Factory
+        {
+            private static Operation Make(Instruction inst, int destCount, int srcCount)
+            {
+                Data* data = Allocators.Operations.Allocate<Data>();
+                *data = default;
+
+                Operation result = new();
+                result._data = data;
+                result.Instruction = inst;
+
+                EnsureCapacity(ref result._data->Destinations, ref result._data->DestinationsCount, destCount);
+                EnsureCapacity(ref result._data->Sources, ref result._data->SourcesCount, srcCount);
+
+                result.Destinations.Clear();
+                result.Sources.Clear();
+
+                return result;
+            }
+
+            public static Operation Operation(Instruction inst, Operand dest)
+            {
+                Operation result = Make(inst, 0, 0);
+                result.SetDestination(dest);
+                return result;
+            }
+
+            public static Operation Operation(Instruction inst, Operand dest, Operand src0)
+            {
+                Operation result = Make(inst, 0, 1);
+                result.SetDestination(dest);
+                result.SetSource(0, src0);
+                return result;
+            }
+
+            public static Operation Operation(Instruction inst, Operand dest, Operand src0, Operand src1)
+            {
+                Operation result = Make(inst, 0, 2);
+                result.SetDestination(dest);
+                result.SetSource(0, src0);
+                result.SetSource(1, src1);
+                return result;
+            }
+
+            public static Operation Operation(Instruction inst, Operand dest, Operand src0, Operand src1, Operand src2)
+            {
+                Operation result = Make(inst, 0, 3);
+                result.SetDestination(dest);
+                result.SetSource(0, src0);
+                result.SetSource(1, src1);
+                result.SetSource(2, src2);
+                return result;
+            }
+
+            public static Operation Operation(Instruction inst, Operand dest, int srcCount)
+            {
+                Operation result = Make(inst, 0, srcCount);
+                result.SetDestination(dest);
+                return result;
+            }
+
+            public static Operation Operation(Instruction inst, Operand dest, Operand[] srcs)
+            {
+                Operation result = Make(inst, 0, srcs.Length);
+
+                result.SetDestination(dest);
+
+                for (int index = 0; index < srcs.Length; index++)
+                {
+                    result.SetSource(index, srcs[index]);
+                }
+
+                return result;
+            }
+
+            public static Operation Operation(Intrinsic intrin, Operand dest, params Operand[] srcs)
+            {
+                Operation result = Make(Instruction.Extended, 0, srcs.Length);
+
+                result.Intrinsic = intrin;
+                result.SetDestination(dest);
+
+                for (int index = 0; index < srcs.Length; index++)
+                {
+                    result.SetSource(index, srcs[index]);
+                }
+
+                return result;
+            }
+
+            public static Operation Operation(Instruction inst, Operand[] dests, Operand[] srcs)
+            {
+                Operation result = Make(inst, dests.Length, srcs.Length);
+
+                for (int index = 0; index < dests.Length; index++)
+                {
+                    result.SetDestination(index, dests[index]);
+                }
+
+                for (int index = 0; index < srcs.Length; index++)
+                {
+                    result.SetSource(index, srcs[index]);
+                }
+
+                return result;
+            }
+
+            public static Operation PhiOperation(Operand dest, int srcCount)
+            {
+                return Operation(Instruction.Phi, dest, srcCount * 2);
+            }
+        }
     }
 }

ARMeilleure/IntermediateRepresentation/OperationHelper.cs

@@ -1,66 +0,0 @@
-using ARMeilleure.Common;
-
-namespace ARMeilleure.IntermediateRepresentation
-{
-    static class OperationHelper
-    {
-        public static Operation Operation(Instruction instruction, Operand destination)
-        {
-            return Operation().With(instruction, destination);
-        }
-
-        public static Operation Operation(Instruction instruction, Operand destination,
-            Operand[] sources)
-        {
-            return Operation().With(instruction, destination, sources);
-        }
-
-        public static Operation Operation(Instruction instruction, Operand destination, 
-            Operand source0)
-        {
-            return Operation().With(instruction, destination, source0);
-        }
-
-        public static Operation Operation(Instruction instruction, Operand destination, 
-            Operand source0, Operand source1)
-        {
-            return Operation().With(instruction, destination, source0, source1);
-        }
-
-        public static Operation Operation(Instruction instruction, Operand destination,
-            Operand source0, Operand source1, Operand source2)
-        {
-            return Operation().With(instruction, destination, source0, source1, source2);
-        }
-
-        public static Operation Operation(
-            Instruction instruction,
-            Operand[] destinations,
-            Operand[] sources)
-        {
-            return Operation().With(instruction, destinations, sources);
-        }
-
-        #region "ThreadStaticPool"
-        public static void PrepareOperationPool(int groupId = 0)
-        {
-            ThreadStaticPool<Operation>.PreparePool(groupId, ChunkSizeLimit.Medium);
-        }
-
-        private static Operation Operation()
-        {
-            return ThreadStaticPool<Operation>.Instance.Allocate();
-        }
-
-        public static void ResetOperationPool(int groupId = 0)
-        {
-            ThreadStaticPool<Operation>.ResetPool(groupId);
-        }
-
-        public static void DisposeOperationPools()
-        {
-            ThreadStaticPool<Operation>.DisposePools();
-        }
-        #endregion
-    }
-}

ARMeilleure/IntermediateRepresentation/PhiNode.cs

@@ -1,22 +0,0 @@
-namespace ARMeilleure.IntermediateRepresentation
-{
-    class PhiNode : Node
-    {
-        private BasicBlock[] _blocks;
-
-        public PhiNode(Operand destination, int predecessorsCount) : base(destination, predecessorsCount)
-        {
-            _blocks = new BasicBlock[predecessorsCount];
-        }
-
-        public BasicBlock GetBlock(int index)
-        {
-            return _blocks[index];
-        }
-
-        public void SetBlock(int index, BasicBlock block)
-        {
-            _blocks[index] = block;
-        }
-    }
-}

ARMeilleure/IntermediateRepresentation/PhiOperation.cs

@@ -0,0 +1,37 @@
+using ARMeilleure.Translation;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
+
+namespace ARMeilleure.IntermediateRepresentation
+{
+    struct PhiOperation
+    {
+        private readonly Operation _operation;
+
+        public PhiOperation(Operation operation)
+        {
+            _operation = operation;
+        }
+
+        public int SourcesCount => _operation.SourcesCount / 2;
+
+        public BasicBlock GetBlock(ControlFlowGraph cfg, int index)
+        {
+            return cfg.PostOrderBlocks[cfg.PostOrderMap[_operation.GetSource(index * 2).AsInt32()]];
+        }
+
+        public void SetBlock(int index, BasicBlock block)
+        {
+            _operation.SetSource(index * 2, Const(block.Index));
+        }
+
+        public Operand GetSource(int index)
+        {
+            return _operation.GetSource(index * 2 + 1);
+        }
+
+        public void SetSource(int index, Operand operand)
+        {
+            _operation.SetSource(index * 2 + 1, operand);
+        }
+    }
+}

ARMeilleure/Signal/NativeSignalHandler.cs

@@ -4,7 +4,7 @@ using System;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Signal
 {
@@ -95,8 +95,6 @@ namespace ARMeilleure.Signal
             {
                 if (_initialized) return;
 
-                Translator.PreparePool();
-
                 bool unix = RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
                 ref SignalHandlerConfig config = ref GetConfigRef();
 
@@ -124,10 +122,6 @@ namespace ARMeilleure.Signal
                     _signalHandlerHandle = WindowsSignalHandlerRegistration.RegisterExceptionHandler(_signalHandlerPtr);
                 }
 
-                Translator.ResetPool();
-
-                Translator.DisposePools();
-
                 _initialized = true;
             }
         }

ARMeilleure/Translation/ArmEmitterContext.cs

@@ -9,7 +9,7 @@ using ARMeilleure.Translation.PTC;
 using System;
 using System.Collections.Generic;
 using System.Reflection;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Translation
 {
@@ -138,7 +138,7 @@ namespace ARMeilleure.Translation
         {
             if (_optOpLastCompare == null || _optOpLastCompare != _optOpLastFlagSet)
             {
-                return null;
+                return default;
             }
 
             Operand n = _optCmpTempN;
@@ -193,7 +193,7 @@ namespace ARMeilleure.Translation
                 }
             }
 
-            return null;
+            return default;
         }
     }
 }

ARMeilleure/Translation/Compiler.cs

@@ -55,7 +55,7 @@ namespace ARMeilleure.Translation
 
             Logger.EndPass(PassName.SsaConstruction, cfg);
 
-            CompilerContext cctx = new CompilerContext(cfg, argTypes, retType, options);
+            CompilerContext cctx = new(cfg, argTypes, retType, options);
 
             return CodeGenerator.Generate(cctx, ptcInfo);
         }

ARMeilleure/Translation/ControlFlowGraph.cs

@@ -22,15 +22,12 @@ namespace ARMeilleure.Translation
             Blocks = blocks;
             LocalsCount = localsCount;
 
-            Update(removeUnreachableBlocks: true);
+            Update();
         }
 
-        public void Update(bool removeUnreachableBlocks)
+        public void Update()
-        {
-            if (removeUnreachableBlocks)
         {
             RemoveUnreachableBlocks(Blocks);
-            }
 
             var visited = new HashSet<BasicBlock>();
             var blockStack = new Stack<BasicBlock>();
@@ -47,7 +44,7 @@ namespace ARMeilleure.Translation
             {
                 bool visitedNew = false;
 
-                for (int i = 0; i < block.SuccessorCount; i++)
+                for (int i = 0; i < block.SuccessorsCount; i++)
                 {
                     BasicBlock succ = block.GetSuccessor(i);
 
@@ -83,7 +80,7 @@ namespace ARMeilleure.Translation
             {
                 Debug.Assert(block.Index != -1, "Invalid block index.");
 
-                for (int i = 0; i < block.SuccessorCount; i++)
+                for (int i = 0; i < block.SuccessorsCount; i++)
                 {
                     BasicBlock succ = block.GetSuccessor(i);
 
@@ -105,9 +102,9 @@ namespace ARMeilleure.Translation
 
                     if (!visited.Contains(block))
                     {
-                        while (block.SuccessorCount > 0)
+                        while (block.SuccessorsCount > 0)
                         {
-                            block.RemoveSuccessor(index: block.SuccessorCount - 1);
+                            block.RemoveSuccessor(index: block.SuccessorsCount - 1);
                         }
 
                         blocks.Remove(block);
@@ -126,7 +123,7 @@ namespace ARMeilleure.Translation
         {
             BasicBlock splitBlock = new BasicBlock(Blocks.Count);
 
-            for (int i = 0; i < predecessor.SuccessorCount; i++)
+            for (int i = 0; i < predecessor.SuccessorsCount; i++)
             {
                 if (predecessor.GetSuccessor(i) == successor)
                 {

ARMeilleure/Translation/EmitterContext.cs

@@ -1,12 +1,10 @@
 using ARMeilleure.Diagnostics;
 using ARMeilleure.IntermediateRepresentation;
 using ARMeilleure.State;
-using ARMeilleure.Translation.PTC;
 using System;
 using System.Collections.Generic;
 using System.Reflection;
-
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
 
 namespace ARMeilleure.Translation
 {
@@ -77,7 +75,7 @@ namespace ARMeilleure.Translation
 
         public void BranchIf(Operand label, Operand op1, Operand op2, Comparison comp, BasicBlockFrequency falseFreq = default)
         {
-            Add(Instruction.BranchIf, null, op1, op2, Const((int)comp));
+            Add(Instruction.BranchIf, default, op1, op2, Const((int)comp));
 
             BranchToLabel(label, uncond: false, falseFreq);
         }
@@ -157,7 +155,7 @@ namespace ARMeilleure.Translation
             }
             else
             {
-                return Add(Instruction.Call, null, args);
+                return Add(Instruction.Call, default, args);
             }
         }
 
@@ -169,7 +167,7 @@ namespace ARMeilleure.Translation
 
             Array.Copy(callArgs, 0, args, 1, callArgs.Length);
 
-            Add(Instruction.Tailcall, null, args);
+            Add(Instruction.Tailcall, default, args);
 
             _needsNewBlock = true;
         }
@@ -356,7 +354,7 @@ namespace ARMeilleure.Translation
 
         public void Return(Operand op1)
         {
-            Add(Instruction.Return, null, op1);
+            Add(Instruction.Return, default, op1);
 
             _needsNewBlock = true;
         }
@@ -398,17 +396,17 @@ namespace ARMeilleure.Translation
 
         public void Store(Operand address, Operand value)
         {
-            Add(Instruction.Store, null, address, value);
+            Add(Instruction.Store, default, address, value);
         }
 
         public void Store16(Operand address, Operand value)
         {
-            Add(Instruction.Store16, null, address, value);
+            Add(Instruction.Store16, default, address, value);
         }
 
         public void Store8(Operand address, Operand value)
         {
-            Add(Instruction.Store8, null, address, value);
+            Add(Instruction.Store8, default, address, value);
         }
 
         public void StoreToContext()
@@ -501,11 +499,11 @@ namespace ARMeilleure.Translation
             }
         }
 
-        private Operand Add(Instruction inst, Operand dest = null)
+        private Operand Add(Instruction inst, Operand dest = default)
         {
             NewNextBlockIfNeeded();
 
-            Operation operation = OperationHelper.Operation(inst, dest);
+            Operation operation = Operation.Factory.Operation(inst, dest);
 
             _irBlock.Operations.AddLast(operation);
 
@@ -516,7 +514,7 @@ namespace ARMeilleure.Translation
         {
             NewNextBlockIfNeeded();
 
-            Operation operation = OperationHelper.Operation(inst, dest, sources);
+            Operation operation = Operation.Factory.Operation(inst, dest, sources);
 
             _irBlock.Operations.AddLast(operation);
 
@@ -527,7 +525,7 @@ namespace ARMeilleure.Translation
         {
             NewNextBlockIfNeeded();
 
-            Operation operation = OperationHelper.Operation(inst, dest, source0);
+            Operation operation = Operation.Factory.Operation(inst, dest, source0);
 
             _irBlock.Operations.AddLast(operation);
 
@@ -538,7 +536,7 @@ namespace ARMeilleure.Translation
         {
             NewNextBlockIfNeeded();
 
-            Operation operation = OperationHelper.Operation(inst, dest, source0, source1);
+            Operation operation = Operation.Factory.Operation(inst, dest, source0, source1);
 
             _irBlock.Operations.AddLast(operation);
 
@@ -549,7 +547,7 @@ namespace ARMeilleure.Translation
         {
             NewNextBlockIfNeeded();
 
-            Operation operation = OperationHelper.Operation(inst, dest, source0, source1, source2);
+            Operation operation = Operation.Factory.Operation(inst, dest, source0, source1, source2);
 
             _irBlock.Operations.AddLast(operation);
 
@@ -573,14 +571,14 @@ namespace ARMeilleure.Translation
 
         public void AddIntrinsicNoRet(Intrinsic intrin, params Operand[] args)
         {
-            Add(intrin, null, args);
+            Add(intrin, default, args);
         }
 
         private Operand Add(Intrinsic intrin, Operand dest, params Operand[] sources)
         {
             NewNextBlockIfNeeded();
 
-            IntrinsicOperation operation = new IntrinsicOperation(intrin, dest, sources);
+            Operation operation = Operation.Factory.Operation(intrin, dest, sources);
 
             _irBlock.Operations.AddLast(operation);
 
@@ -641,7 +639,7 @@ namespace ARMeilleure.Translation
 
         private void NextBlock(BasicBlock nextBlock)
         {
-            if (_irBlock?.SuccessorCount == 0 && !EndsWithUnconditional(_irBlock))
+            if (_irBlock?.SuccessorsCount == 0 && !EndsWithUnconditional(_irBlock))
             {
                 _irBlock.AddSuccessor(nextBlock);
 
@@ -662,9 +660,11 @@ namespace ARMeilleure.Translation
 
         private static bool EndsWithUnconditional(BasicBlock block)
         {
-            return block.Operations.Last is Operation lastOp &&
+            Operation last = block.Operations.Last;
-                   (lastOp.Instruction == Instruction.Return ||
+
-                    lastOp.Instruction == Instruction.Tailcall);
+            return last != default &&
+               (last.Instruction == Instruction.Return ||
+                last.Instruction == Instruction.Tailcall);
         }
 
         public ControlFlowGraph GetControlFlowGraph()

ARMeilleure/Translation/PTC/DegreeOfParallelism.cs

@@ -1,50 +0,0 @@
-using System;
-
-namespace ARMeilleure.Translation.PTC
-{
-    class DegreeOfParallelism
-    {
-        public double GiBRef { get; } // GiB.
-        public double WeightRef { get; } // %.
-        public double IncrementByGiB { get; } // %.
-        private double _coefficient;
-
-        public DegreeOfParallelism(double gibRef, double weightRef, double incrementByGiB)
-        {
-            GiBRef = gibRef;
-            WeightRef = weightRef;
-            IncrementByGiB = incrementByGiB;
-
-            _coefficient = weightRef - (incrementByGiB * gibRef);
-        }
-
-        public int GetDegreeOfParallelism(int min, int max)
-        {
-            double degreeOfParallelism = (GetProcessorCount() * GetWeight(GetAvailableMemoryGiB())) / 100d;
-
-            return Math.Clamp((int)Math.Round(degreeOfParallelism), min, max);
-        }
-
-        public static double GetProcessorCount()
-        {
-            return (double)Environment.ProcessorCount;
-        }
-
-        public double GetWeight(double gib)
-        {
-            return (IncrementByGiB * gib) + _coefficient;
-        }
-
-        public static double GetAvailableMemoryGiB()
-        {
-            GCMemoryInfo gcMemoryInfo = GC.GetGCMemoryInfo();
-
-            return FromBytesToGiB(gcMemoryInfo.TotalAvailableMemoryBytes - gcMemoryInfo.MemoryLoadBytes);
-        }
-
-        private static double FromBytesToGiB(long bytes)
-        {
-            return Math.ScaleB((double)bytes, -30);
-        }
-    }
-}

ARMeilleure/Translation/PTC/Ptc.cs

@@ -9,7 +9,6 @@ using Ryujinx.Common.Configuration;
 using Ryujinx.Common.Logging;
 using System;
 using System.Buffers.Binary;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -28,7 +27,7 @@ namespace ARMeilleure.Translation.PTC
         private const string OuterHeaderMagicString = "PTCohd\0\0";
         private const string InnerHeaderMagicString = "PTCihd\0\0";
 
-        private const uint InternalVersion = 2228; //! To be incremented manually for each change to the ARMeilleure project.
+        private const uint InternalVersion = 2515; //! To be incremented manually for each change to the ARMeilleure project.
 
         private const string ActualDir = "0";
         private const string BackupDir = "1";
@@ -776,9 +775,7 @@ namespace ARMeilleure.Translation.PTC
             _translateCount = 0;
             _translateTotalCount = profiledFuncsToTranslate.Count;
 
-            int degreeOfParallelism = new DegreeOfParallelism(4d, 75d, 12.5d).GetDegreeOfParallelism(0, 32);
+            if (_translateTotalCount == 0)
-
-            if (_translateTotalCount == 0 || degreeOfParallelism == 0)
             {
                 ResetCarriersIfNeeded();
 
@@ -787,6 +784,14 @@ namespace ARMeilleure.Translation.PTC
                 return;
             }
 
+            int degreeOfParallelism = Environment.ProcessorCount;
+
+            // If there are enough cores lying around, we leave one alone for other tasks.
+            if (degreeOfParallelism > 4)
+            {
+                degreeOfParallelism--;
+            }
+
             Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}");
 
             PtcStateChanged?.Invoke(PtcLoadingState.Start, _translateCount, _translateTotalCount);
@@ -825,8 +830,6 @@ namespace ARMeilleure.Translation.PTC
                         break;
                     }
                 }
-
-                Translator.DisposePools();
             }
 
             List<Thread> threads = new List<Thread>();
@@ -839,6 +842,8 @@ namespace ARMeilleure.Translation.PTC
                 threads.Add(thread);
             }
 
+            Stopwatch sw = Stopwatch.StartNew();
+
             threads.ForEach((thread) => thread.Start());
             threads.ForEach((thread) => thread.Join());
 
@@ -847,9 +852,11 @@ namespace ARMeilleure.Translation.PTC
             progressReportEvent.Set();
             progressReportThread.Join();
 
+            sw.Stop();
+
             PtcStateChanged?.Invoke(PtcLoadingState.Loaded, _translateCount, _translateTotalCount);
 
-            Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}");
+            Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s");
 
             Thread preSaveThread = new Thread(PreSave);
             preSaveThread.IsBackground = true;

ARMeilleure/Translation/RegisterToLocal.cs

@@ -1,7 +1,7 @@
 using ARMeilleure.IntermediateRepresentation;
 using System.Collections.Generic;
 
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Translation
 {
@@ -27,11 +27,11 @@ namespace ARMeilleure.Translation
 
             for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext)
             {
-                for (Node node = block.Operations.First; node != null; node = node.ListNext)
+                for (Operation node = block.Operations.First; node != default; node = node.ListNext)
                 {
                     Operand dest = node.Destination;
 
-                    if (dest != null && dest.Kind == OperandKind.Register)
+                    if (dest != default && dest.Kind == OperandKind.Register)
                     {
                         node.Destination = GetLocal(dest);
                     }

ARMeilleure/Translation/RegisterUsage.cs

@@ -1,9 +1,11 @@
 using ARMeilleure.IntermediateRepresentation;
 using ARMeilleure.State;
 using System;
-
+using System.Numerics;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using System.Runtime.Intrinsics;
-using static ARMeilleure.IntermediateRepresentation.OperationHelper;
+using System.Runtime.Intrinsics.X86;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
+using static ARMeilleure.IntermediateRepresentation.Operation.Factory;
 
 namespace ARMeilleure.Translation
 {
@@ -14,27 +16,48 @@ namespace ARMeilleure.Translation
 
         private struct RegisterMask : IEquatable<RegisterMask>
         {
-            public long IntMask { get; set; }
+            public long IntMask => Mask.GetElement(0);
-            public long VecMask { get; set; }
+            public long VecMask => Mask.GetElement(1);
+
+            public Vector128<long> Mask { get; }
+
+            public RegisterMask(Vector128<long> mask)
+            {
+                Mask = mask;
+            }
 
             public RegisterMask(long intMask, long vecMask)
             {
-                IntMask = intMask;
+                Mask = Vector128.Create(intMask, vecMask);
-                VecMask = vecMask;
             }
 
             public static RegisterMask operator &(RegisterMask x, RegisterMask y)
             {
+                if (Sse2.IsSupported)
+                {
+                    return new RegisterMask(Sse2.And(x.Mask, y.Mask));
+                }
+
                 return new RegisterMask(x.IntMask & y.IntMask, x.VecMask & y.VecMask);
             }
 
             public static RegisterMask operator |(RegisterMask x, RegisterMask y)
             {
+                if (Sse2.IsSupported)
+                {
+                    return new RegisterMask(Sse2.Or(x.Mask, y.Mask));
+                }
+
                 return new RegisterMask(x.IntMask | y.IntMask, x.VecMask | y.VecMask);
             }
 
             public static RegisterMask operator ~(RegisterMask x)
             {
+                if (Sse2.IsSupported)
+                {
+                    return new RegisterMask(Sse2.AndNot(x.Mask, Vector128<long>.AllBitsSet));
+                }
+
                 return new RegisterMask(~x.IntMask, ~x.VecMask);
             }
 
@@ -55,12 +78,12 @@ namespace ARMeilleure.Translation
 
             public bool Equals(RegisterMask other)
             {
-                return IntMask == other.IntMask && VecMask == other.VecMask;
+                return Mask.Equals(other.Mask);
             }
 
             public override int GetHashCode()
             {
-                return HashCode.Combine(IntMask, VecMask);
+                return Mask.GetHashCode();
             }
         }
 
@@ -72,27 +95,23 @@ namespace ARMeilleure.Translation
 
             for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext)
             {
-                for (Node node = block.Operations.First; node != null; node = node.ListNext)
+                for (Operation node = block.Operations.First; node != default; node = node.ListNext)
                 {
-                    Operation operation = node as Operation;
+                    for (int index = 0; index < node.SourcesCount; index++)
-
-                    for (int srcIndex = 0; srcIndex < operation.SourcesCount; srcIndex++)
                     {
-                        Operand source = operation.GetSource(srcIndex);
+                        Operand source = node.GetSource(index);
 
-                        if (source.Kind != OperandKind.Register)
+                        if (source.Kind == OperandKind.Register)
                         {
-                            continue;
-                        }
-
                             Register register = source.GetRegister();
 
                             localInputs[block.Index] |= GetMask(register) & ~localOutputs[block.Index];
                         }
+                    }
 
-                    if (operation.Destination != null && operation.Destination.Kind == OperandKind.Register)
+                    if (node.Destination != default && node.Destination.Kind == OperandKind.Register)
                     {
-                        localOutputs[block.Index] |= GetMask(operation.Destination.GetRegister());
+                        localOutputs[block.Index] |= GetMask(node.Destination.GetRegister());
                     }
                 }
             }
@@ -104,7 +123,6 @@ namespace ARMeilleure.Translation
             RegisterMask[] globalOutputs = new RegisterMask[cfg.Blocks.Count];
 
             bool modified;
-
             bool firstPass = true;
 
             do
@@ -121,7 +139,6 @@ namespace ARMeilleure.Translation
                         BasicBlock predecessor = block.Predecessors[0];
 
                         RegisterMask cmnOutputs = localOutputs[predecessor.Index] | globalCmnOutputs[predecessor.Index];
-
                         RegisterMask outputs = globalOutputs[predecessor.Index];
 
                         for (int pIndex = 1; pIndex < block.Predecessors.Count; pIndex++)
@@ -129,7 +146,6 @@ namespace ARMeilleure.Translation
                             predecessor = block.Predecessors[pIndex];
 
                             cmnOutputs &= localOutputs[predecessor.Index] | globalCmnOutputs[predecessor.Index];
-
                             outputs |= globalOutputs[predecessor.Index];
                         }
 
@@ -140,21 +156,13 @@ namespace ARMeilleure.Translation
                             cmnOutputs &= globalCmnOutputs[block.Index];
                         }
 
-                        if (Exchange(globalCmnOutputs, block.Index, cmnOutputs))
+                        modified |= Exchange(globalCmnOutputs, block.Index, cmnOutputs);
-                        {
-                            modified = true;
-                        }
-
                         outputs |= localOutputs[block.Index];
-
+                        modified |= Exchange(globalOutputs, block.Index, globalOutputs[block.Index] | outputs);
-                        if (Exchange(globalOutputs, block.Index, globalOutputs[block.Index] | outputs))
-                        {
-                            modified = true;
                     }
-                    }
+                    else
-                    else if (Exchange(globalOutputs, block.Index, localOutputs[block.Index]))
                     {
-                        modified = true;
+                        modified |= Exchange(globalOutputs, block.Index, localOutputs[block.Index]);
                     }
                 }
 
@@ -165,17 +173,14 @@ namespace ARMeilleure.Translation
 
                     RegisterMask inputs = localInputs[block.Index];
 
-                    for (int i = 0; i < block.SuccessorCount; i++)
+                    for (int i = 0; i < block.SuccessorsCount; i++)
                     {
                         inputs |= globalInputs[block.GetSuccessor(i).Index];
                     }
 
                     inputs &= ~globalCmnOutputs[block.Index];
 
-                    if (Exchange(globalInputs, block.Index, globalInputs[block.Index] | inputs))
+                    modified |= Exchange(globalInputs, block.Index, globalInputs[block.Index] | inputs);
-                    {
-                        modified = true;
-                    }
                 }
 
                 firstPass = false;
@@ -192,12 +197,18 @@ namespace ARMeilleure.Translation
                     block.Operations.Remove(block.Operations.First);
                 }
 
+                Operand arg = default;
+
                 // The only block without any predecessor should be the entry block.
                 // It always needs a context load as it is the first block to run.
                 if (block.Predecessors.Count == 0 || hasContextLoad)
                 {
-                    LoadLocals(block, globalInputs[block.Index].VecMask, RegisterType.Vector,  mode);
+                    arg = Local(OperandType.I64);
-                    LoadLocals(block, globalInputs[block.Index].IntMask, RegisterType.Integer, mode);
+
+                    Operation loadArg = block.Operations.AddFirst(Operation(Instruction.LoadArgument, arg, Const(0)));
+
+                    LoadLocals(block, globalInputs[block.Index].VecMask, RegisterType.Vector, mode, loadArg, arg);
+                    LoadLocals(block, globalInputs[block.Index].IntMask, RegisterType.Integer, mode, loadArg, arg);
                 }
 
                 bool hasContextStore = HasContextStore(block);
@@ -209,8 +220,15 @@ namespace ARMeilleure.Translation
 
                 if (EndsWithReturn(block) || hasContextStore)
                 {
-                    StoreLocals(block, globalOutputs[block.Index].IntMask, RegisterType.Integer, mode);
+                    if (arg == default)
-                    StoreLocals(block, globalOutputs[block.Index].VecMask, RegisterType.Vector,  mode);
+                    {
+                        arg = Local(OperandType.I64);
+
+                        block.Append(Operation(Instruction.LoadArgument, arg, Const(0)));
+                    }
+
+                    StoreLocals(block, globalOutputs[block.Index].IntMask, RegisterType.Integer, mode, arg);
+                    StoreLocals(block, globalOutputs[block.Index].VecMask, RegisterType.Vector, mode, arg);
                 }
             }
         }
@@ -222,27 +240,31 @@ namespace ARMeilleure.Translation
 
         private static bool HasContextStore(BasicBlock block)
         {
-            return EndsWith(block, Instruction.StoreToContext) && block.GetLastOp().SourcesCount == 0;
+            return EndsWith(block, Instruction.StoreToContext) && block.Operations.Last.SourcesCount == 0;
         }
 
         private static bool StartsWith(BasicBlock block, Instruction inst)
         {
-            if (block.Operations.Count == 0)
+            if (block.Operations.Count > 0)
             {
-                return false;
+                Operation first = block.Operations.First;
+
+                return first != default && first.Instruction == inst;
             }
 
-            return block.Operations.First is Operation operation && operation.Instruction == inst;
+            return false;
         }
 
         private static bool EndsWith(BasicBlock block, Instruction inst)
         {
-            if (block.Operations.Count == 0)
+            if (block.Operations.Count > 0)
             {
-                return false;
+                Operation last = block.Operations.Last;
+
+                return last != default && last.Instruction == inst;
             }
 
-            return block.Operations.Last is Operation operation && operation.Instruction == inst;
+            return false;
         }
 
         private static RegisterMask GetMask(Register register)
@@ -263,76 +285,57 @@ namespace ARMeilleure.Translation
 
         private static bool Exchange(RegisterMask[] masks, int blkIndex, RegisterMask value)
         {
-            RegisterMask oldValue = masks[blkIndex];
+            ref RegisterMask curValue = ref masks[blkIndex];
 
-            masks[blkIndex] = value;
+            bool changed = curValue != value;
 
-            return oldValue != value;
+            curValue = value;
+
+            return changed;
         }
 
-        private static void LoadLocals(BasicBlock block, long inputs, RegisterType baseType, ExecutionMode mode)
+        private static void LoadLocals(
+            BasicBlock block,
+            long inputs,
+            RegisterType baseType,
+            ExecutionMode mode,
+            Operation loadArg,
+            Operand arg)
         {
-            Operand arg0 = Local(OperandType.I64);
+            while (inputs != 0)
-
-            for (int bit = 63; bit >= 0; bit--)
             {
-                long mask = 1L << bit;
+                int bit = 63 - BitOperations.LeadingZeroCount((ulong)inputs);
-
-                if ((inputs & mask) == 0)
-                {
-                    continue;
-                }
 
                 Operand dest = GetRegFromBit(bit, baseType, mode);
-
+                Operand offset = Const((long)NativeContext.GetRegisterOffset(dest.GetRegister()));
-                long offset = NativeContext.GetRegisterOffset(dest.GetRegister());
-
                 Operand addr = Local(OperandType.I64);
 
-                Operation loadOp = Operation(Instruction.Load, dest, addr);
+                block.Operations.AddAfter(loadArg, Operation(Instruction.Load, dest, addr));
+                block.Operations.AddAfter(loadArg, Operation(Instruction.Add, addr, arg, offset));
 
-                block.Operations.AddFirst(loadOp);
+                inputs &= ~(1L << bit);
-
+            }
-                Operation calcOffsOp = Operation(Instruction.Add, addr, arg0, Const(offset));
-
-                block.Operations.AddFirst(calcOffsOp);
         }
 
-            Operation loadArg0 = Operation(Instruction.LoadArgument, arg0, Const(0));
+        private static void StoreLocals(
-
+            BasicBlock block,
-            block.Operations.AddFirst(loadArg0);
+            long outputs,
-        }
+            RegisterType baseType,
-
+            ExecutionMode mode,
-        private static void StoreLocals(BasicBlock block, long outputs, RegisterType baseType, ExecutionMode mode)
+            Operand arg)
         {
-            Operand arg0 = Local(OperandType.I64);
+            while (outputs != 0)
-
-            Operation loadArg0 = Operation(Instruction.LoadArgument, arg0, Const(0));
-
-            block.Append(loadArg0);
-
-            for (int bit = 0; bit < 64; bit++)
             {
-                long mask = 1L << bit;
+                int bit = BitOperations.TrailingZeroCount(outputs);
-
-                if ((outputs & mask) == 0)
-                {
-                    continue;
-                }
 
                 Operand source = GetRegFromBit(bit, baseType, mode);
-
+                Operand offset = Const((long)NativeContext.GetRegisterOffset(source.GetRegister()));
-                long offset = NativeContext.GetRegisterOffset(source.GetRegister());
-
                 Operand addr = Local(OperandType.I64);
 
-                Operation calcOffsOp = Operation(Instruction.Add, addr, arg0, Const(offset));
+                block.Append(Operation(Instruction.Add, addr, arg, offset));
+                block.Append(Operation(Instruction.Store, default, addr, source));
 
-                block.Append(calcOffsOp);
+                outputs &= ~(1L << bit);
-
-                Operation storeOp = Operation(Instruction.Store, null, addr, source);
-
-                block.Append(storeOp);
             }
         }
 
@@ -340,15 +343,15 @@ namespace ARMeilleure.Translation
         {
             if (bit < RegsCount)
             {
-                return OperandHelper.Register(bit, baseType, GetOperandType(baseType, mode));
+                return Register(bit, baseType, GetOperandType(baseType, mode));
             }
             else if (baseType == RegisterType.Integer)
             {
-                return OperandHelper.Register(bit & RegsMask, RegisterType.Flag, OperandType.I32);
+                return Register(bit & RegsMask, RegisterType.Flag, OperandType.I32);
             }
             else if (baseType == RegisterType.Vector)
             {
-                return OperandHelper.Register(bit & RegsMask, RegisterType.FpFlag, OperandType.I32);
+                return Register(bit & RegsMask, RegisterType.FpFlag, OperandType.I32);
             }
             else
             {
@@ -371,12 +374,9 @@ namespace ARMeilleure.Translation
 
         private static bool EndsWithReturn(BasicBlock block)
         {
-            if (!(block.GetLastOp() is Operation operation))
+            Operation last = block.Operations.Last;
-            {
-                return false;
-            }
 
-            return operation.Instruction == Instruction.Return;
+            return last != default && last.Instruction == Instruction.Return;
         }
     }
 }

ARMeilleure/Translation/SsaConstruction.cs

@@ -4,7 +4,7 @@ using ARMeilleure.State;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Translation
 {
@@ -18,7 +18,7 @@ namespace ARMeilleure.Translation
             public DefMap()
             {
                 _map = new Dictionary<int, Operand>();
-                _phiMasks = new BitMap(RegisterConsts.TotalCount);
+                _phiMasks = new BitMap(Allocators.Default, RegisterConsts.TotalCount);
             }
 
             public bool TryAddOperand(int key, Operand operand)
@@ -57,26 +57,26 @@ namespace ARMeilleure.Translation
             // First pass, get all defs and locals uses.
             for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext)
             {
-                for (Node node = block.Operations.First; node != null; node = node.ListNext)
+                for (Operation node = block.Operations.First; node != default; node = node.ListNext)
                 {
-                    if (node is not Operation operation)
+                    for (int index = 0; index < node.SourcesCount; index++)
                     {
-                        continue;
+                        Operand src = node.GetSource(index);
-                    }
-
-                    for (int index = 0; index < operation.SourcesCount; index++)
-                    {
-                        Operand src = operation.GetSource(index);
 
                         if (TryGetId(src, out int srcKey))
                         {
-                            Operand local = localDefs[srcKey] ?? src;
+                            Operand local = localDefs[srcKey];
 
-                            operation.SetSource(index, local);
+                            if (local == default)
+                            {
+                                local = src;
+                            }
+
+                            node.SetSource(index, local);
                         }
                     }
 
-                    Operand dest = operation.Destination;
+                    Operand dest = node.Destination;
 
                     if (TryGetId(dest, out int destKey))
                     {
@@ -84,7 +84,7 @@ namespace ARMeilleure.Translation
 
                         localDefs[destKey] = local;
 
-                        operation.Destination = local;
+                        node.Destination = local;
                     }
                 }
 
@@ -92,7 +92,7 @@ namespace ARMeilleure.Translation
                 {
                     Operand local = localDefs[key];
 
-                    if (local is null)
+                    if (local == default)
                     {
                         continue;
                     }
@@ -119,28 +119,23 @@ namespace ARMeilleure.Translation
             // Second pass, rename variables with definitions on different blocks.
             for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext)
             {
-                for (Node node = block.Operations.First; node != null; node = node.ListNext)
+                for (Operation node = block.Operations.First; node != default; node = node.ListNext)
                 {
-                    if (node is not Operation operation)
+                    for (int index = 0; index < node.SourcesCount; index++)
                     {
-                        continue;
+                        Operand src = node.GetSource(index);
-                    }
-
-                    for (int index = 0; index < operation.SourcesCount; index++)
-                    {
-                        Operand src = operation.GetSource(index);
 
                         if (TryGetId(src, out int key))
                         {
                             Operand local = localDefs[key];
 
-                            if (local is null)
+                            if (local == default)
                             {
                                 local = FindDef(globalDefs, block, src);
                                 localDefs[key] = local;
                             }
 
-                            operation.SetSource(index, local);
+                            node.SetSource(index, local);
                         }
                     }
                 }
@@ -200,12 +195,14 @@ namespace ARMeilleure.Translation
             // then use the definition from that Phi.
             Operand local = Local(operand.Type);
 
-            PhiNode phi = new PhiNode(local, block.Predecessors.Count);
+            Operation operation = Operation.Factory.PhiOperation(local, block.Predecessors.Count);
 
-            AddPhi(block, phi);
+            AddPhi(block, operation);
 
             globalDefs[block.Index].TryAddOperand(GetId(operand), local);
 
+            PhiOperation phi = operation.AsPhi();
+
             for (int index = 0; index < block.Predecessors.Count; index++)
             {
                 BasicBlock predecessor = block.Predecessors[index];
@@ -217,19 +214,19 @@ namespace ARMeilleure.Translation
             return local;
         }
 
-        private static void AddPhi(BasicBlock block, PhiNode phi)
+        private static void AddPhi(BasicBlock block, Operation phi)
         {
-            Node node = block.Operations.First;
+            Operation node = block.Operations.First;
 
-            if (node != null)
+            if (node != default)
             {
-                while (node.ListNext is PhiNode)
+                while (node.ListNext != default && node.ListNext.Instruction == Instruction.Phi)
                 {
                     node = node.ListNext;
                 }
             }
 
-            if (node is PhiNode)
+            if (node != default && node.Instruction == Instruction.Phi)
             {
                 block.Operations.AddAfter(node, phi);
             }
@@ -241,7 +238,9 @@ namespace ARMeilleure.Translation
 
         private static bool TryGetId(Operand operand, out int result)
         {
-            if (operand is { Kind: OperandKind.Register })
+            if (operand != default)
+            {
+                if (operand.Kind == OperandKind.Register)
                 {
                     Register reg = operand.GetRegister();
 
@@ -264,12 +263,13 @@ namespace ARMeilleure.Translation
 
                     return true;
                 }
-            else if (operand is { Kind: OperandKind.LocalVariable } && operand.GetLocalNumber() > 0)
+                else if (operand.Kind == OperandKind.LocalVariable && operand.GetLocalNumber() > 0)
                 {
                     result = RegisterConsts.TotalCount + operand.GetLocalNumber() - 1;
 
                     return true;
                 }
+            }
 
             result = -1;
 

ARMeilleure/Translation/SsaDeconstruction.cs

@@ -1,7 +1,7 @@
 using ARMeilleure.IntermediateRepresentation;
 
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.OperationHelper;
+using static ARMeilleure.IntermediateRepresentation.Operation.Factory;
 
 namespace ARMeilleure.Translation
 {
@@ -11,34 +11,36 @@ namespace ARMeilleure.Translation
         {
             for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext)
             {
-                Node node = block.Operations.First;
+                Operation operation = block.Operations.First;
 
-                while (node is PhiNode phi)
+                while (operation != default && operation.Instruction == Instruction.Phi)
                 {
-                    Node nextNode = node.ListNext;
+                    Operation nextNode = operation.ListNext;
 
-                    Operand local = Local(phi.Destination.Type);
+                    Operand local = Local(operation.Destination.Type);
+
+                    PhiOperation phi = operation.AsPhi();
 
                     for (int index = 0; index < phi.SourcesCount; index++)
                     {
-                        BasicBlock predecessor = phi.GetBlock(index);
+                        BasicBlock predecessor = phi.GetBlock(cfg, index);
 
                         Operand source = phi.GetSource(index);
 
                         predecessor.Append(Operation(Instruction.Copy, local, source));
 
-                        phi.SetSource(index, null);
+                        phi.SetSource(index, default);
                     }
 
-                    Operation copyOp = Operation(Instruction.Copy, phi.Destination, local);
+                    Operation copyOp = Operation(Instruction.Copy, operation.Destination, local);
 
-                    block.Operations.AddBefore(node, copyOp);
+                    block.Operations.AddBefore(operation, copyOp);
 
-                    phi.Destination = null;
+                    operation.Destination = default;
 
-                    block.Operations.Remove(node);
+                    block.Operations.Remove(operation);
 
-                    node = nextNode;
+                    operation = nextNode;
                 }
             }
         }

ARMeilleure/Translation/Translator.cs

@@ -13,12 +13,8 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Runtime;
 using System.Threading;
-
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.Common.BitMapPool;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
-using static ARMeilleure.IntermediateRepresentation.OperationHelper;
 
 namespace ARMeilleure.Translation
 {
@@ -193,13 +189,9 @@ namespace ARMeilleure.Translation
 
                 ClearJitCache();
 
-                DisposePools();
-
                 Stubs.Dispose();
                 FunctionTable.Dispose();
                 CountTable.Dispose();
-
-                GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
             }
         }
 
@@ -266,8 +258,6 @@ namespace ARMeilleure.Translation
 
             Logger.EndPass(PassName.Decoding);
 
-            PreparePool(highCq ? 1 : 0);
-
             Logger.StartPass(PassName.Translation);
 
             EmitSynchronization(context);
@@ -281,7 +271,7 @@ namespace ARMeilleure.Translation
 
             ulong funcSize = funcRange.End - funcRange.Start;
 
-            Logger.EndPass(PassName.Translation);
+            Logger.EndPass(PassName.Translation, cfg);
 
             Logger.StartPass(PassName.RegisterUsage);
 
@@ -298,8 +288,6 @@ namespace ARMeilleure.Translation
             if (!context.HasPtc)
             {
                 func = Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, options);
-
-                ResetPool(highCq ? 1 : 0);
             }
             else
             {
@@ -307,33 +295,16 @@ namespace ARMeilleure.Translation
 
                 func = Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, options, ptcInfo);
 
-                ResetPool(highCq ? 1 : 0);
-
                 Hash128 hash = Ptc.ComputeHash(Memory, address, funcSize);
 
                 Ptc.WriteInfoCodeRelocUnwindInfo(address, funcSize, hash, highCq, ptcInfo);
             }
 
-            return new TranslatedFunction(func, counter, funcSize, highCq);
+            var result = new TranslatedFunction(func, counter, funcSize, highCq);
-        }
 
-        internal static void PreparePool(int groupId = 0)
+            Allocators.ResetAll();
-        {
-            PrepareOperandPool(groupId);
-            PrepareOperationPool(groupId);
-        }
 
-        internal static void ResetPool(int groupId = 0)
+            return result;
-        {
-            ResetOperationPool(groupId);
-            ResetOperandPool(groupId);
-        }
-
-        internal static void DisposePools()
-        {
-            DisposeOperandPools();
-            DisposeOperationPools();
-            DisposeBitMapPools();
         }
 
         private struct Range
@@ -408,7 +379,7 @@ namespace ARMeilleure.Translation
                             EmitSynchronization(context);
                         }
 
-                        Operand lblPredicateSkip = null;
+                        Operand lblPredicateSkip = default;
 
                         if (opCode is OpCode32 op && op.Cond < Condition.Al)
                         {
@@ -426,7 +397,7 @@ namespace ARMeilleure.Translation
                             throw new InvalidOperationException($"Invalid instruction \"{opCode.Instruction.Name}\".");
                         }
 
-                        if (lblPredicateSkip != null)
+                        if (lblPredicateSkip != default)
                         {
                             context.MarkLabel(lblPredicateSkip);
                         }

ARMeilleure/Translation/TranslatorStubs.cs

@@ -5,7 +5,7 @@ using ARMeilleure.Translation.Cache;
 using System;
 using System.Reflection;
 using System.Runtime.InteropServices;
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
 namespace ARMeilleure.Translation
 {
@@ -145,7 +145,7 @@ namespace ARMeilleure.Translation
             Operand masked = context.BitwiseAnd(guestAddress, Const(~_translator.FunctionTable.Mask));
             context.BranchIfTrue(lblFallback, masked);
 
-            Operand index = null;
+            Operand index = default;
             Operand page = Const((long)_translator.FunctionTable.Base);
 
             for (int i = 0; i < _translator.FunctionTable.Levels.Length; i++)