Add multi-level function table (#2228)

* Add AddressTable<T>

* Use AddressTable<T> for dispatch

* Remove JumpTable & co.

* Add fallback for out of range addresses

* Add PPTC support

* Add documentation to `AddressTable<T>`

* Make AddressTable<T> configurable

* Fix table walk

* Fix IsMapped check

* Remove CountTableCapacity

* Add PPTC support for fast path

* Rename IsMapped to IsValid

* Remove stale comment

* Change format of address in exception message

* Add TranslatorStubs

* Split DispatchStub

Avoids recompilation of stubs during tests.

* Add hint for 64bit or 32bit

* Add documentation to `Symbol`

* Add documentation to `TranslatorStubs`

Make `TranslatorStubs` disposable as well.

* Add documentation to `SymbolType`

* Add `AddressTableEventSource` to monitor function table size

Add an EventSource which measures the amount of unmanaged bytes
allocated by AddressTable<T> instances.

 dotnet-counters monitor -n Ryujinx --counters ARMeilleure

* Add `AllowLcqInFunctionTable` optimization toggle

This is to reduce the impact this change has on the test duration.
Before everytime a test was ran, the FunctionTable would be initialized
and populated so that the newly compiled test would get registered to
it.

* Implement unmanaged dispatcher

Uses the DispatchStub to dispatch into the next translation, which
allows execution to stay in unmanaged for longer and skips a
ConcurrentDictionary look up when the target translation has been
registered to the FunctionTable.

* Remove redundant null check

* Tune levels of FunctionTable

Uses 5 levels instead of 4 and change unit of AddressTableEventSource
from KB to MB.

* Use 64-bit function table

Improves codegen for direct branches:

    mov qword [rax+0x408],0x10603560
 -  mov rcx,sub_10603560_OFFSET
 -  mov ecx,[rcx]
 -  mov ecx,ecx
 -  mov rdx,JIT_CACHE_BASE
 -  add rdx,rcx
 +  mov rcx,sub_10603560
 +  mov rdx,[rcx]
    mov rcx,rax

Improves codegen for dispatch stub:

    and rax,byte +0x1f
 -  mov eax,[rcx+rax*4]
 -  mov eax,eax
 -  mov rcx,JIT_CACHE_BASE
 -  lea rax,[rcx+rax]
 +  mov rax,[rcx+rax*8]
    mov rcx,rbx

* Remove `JitCacheSymbol` & `JitCache.Offset`

* Turn `Translator.Translate` into an instance method

We do not have to add more parameter to this method and related ones as
new structures are added & needed for translation.

* Add symbol only when PTC is enabled

Address LDj3SNuD's feedback

* Change `NativeContext.Running` to a 32-bit integer

* Fix PageTable symbol for host mapped

ARMeilleure/CodeGen/X86/Assembler.cs

@@ -963,8 +963,6 @@ namespace ARMeilleure.CodeGen.X86
                     }
                     else if (dest?.Kind == OperandKind.Register && info.OpRImm64 != BadOp)
                     {
-                        int? index = source.PtcIndex;
-
                         int rexPrefix = GetRexPrefix(dest, source, type, rrm: false);
 
                         if (rexPrefix != 0)
@@ -974,9 +972,9 @@ namespace ARMeilleure.CodeGen.X86
 
                         WriteByte((byte)(info.OpRImm64 + (dest.GetRegister().Index & 0b111)));
 
-                        if (_ptcInfo != null && index != null)
+                        if (_ptcInfo != null && source.Relocatable)
                         {
-                            _ptcInfo.WriteRelocEntry(new RelocEntry((int)_stream.Position, (int)index));
+                            _ptcInfo.WriteRelocEntry(new RelocEntry((int)_stream.Position, source.Symbol));
                         }
 
                         WriteUInt64(imm);

ARMeilleure/Common/AddressTable.cs

@@ -0,0 +1,261 @@
+using ARMeilleure.Diagnostics.EventSources;
+using System;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace ARMeilleure.Common
+{
+    /// <summary>
+    /// Represents a table of guest address to a value.
+    /// </summary>
+    /// <typeparam name="TEntry">Type of the value</typeparam>
+    unsafe class AddressTable<TEntry> : IDisposable where TEntry : unmanaged
+    {
+        /// <summary>
+        /// Represents a level in an <see cref="AddressTable{TEntry}"/>.
+        /// </summary>
+        public readonly struct Level
+        {
+            /// <summary>
+            /// Gets the index of the <see cref="Level"/> in the guest address.
+            /// </summary>
+            public int Index { get; }
+
+            /// <summary>
+            /// Gets the length of the <see cref="Level"/> in the guest address.
+            /// </summary>
+            public int Length { get; }
+
+            /// <summary>
+            /// Gets the mask which masks the bits used by the <see cref="Level"/>.
+            /// </summary>
+            public ulong Mask => ((1ul << Length) - 1) << Index;
+
+            /// <summary>
+            /// Initializes a new instance of the <see cref="Level"/> structure with the specified
+            /// <paramref name="index"/> and <paramref name="length"/>.
+            /// </summary>
+            /// <param name="index">Index of the <see cref="Level"/></param>
+            /// <param name="length">Length of the <see cref="Level"/></param>
+            public Level(int index, int length)
+            {
+                (Index, Length) = (index, length);
+            }
+
+            /// <summary>
+            /// Gets the value of the <see cref="Level"/> from the specified guest <paramref name="address"/>.
+            /// </summary>
+            /// <param name="address">Guest address</param>
+            /// <returns>Value of the <see cref="Level"/> from the specified guest <paramref name="address"/></returns>
+            public int GetValue(ulong address)
+            {
+                return (int)((address & Mask) >> Index);
+            }
+        }
+
+        private bool _disposed;
+        private TEntry** _table;
+        private readonly List<IntPtr> _pages;
+
+        /// <summary>
+        /// Gets the bits used by the <see cref="Levels"/> of the <see cref="AddressTable{TEntry}"/> instance.
+        /// </summary>
+        public ulong Mask { get; }
+
+        /// <summary>
+        /// Gets the <see cref="Level"/>s used by the <see cref="AddressTable{TEntry}"/> instance.
+        /// </summary>
+        public Level[] Levels { get; }
+
+        /// <summary>
+        /// Gets or sets the default fill value of newly created leaf pages.
+        /// </summary>
+        public TEntry Fill { get; set; }
+
+        /// <summary>
+        /// Gets the base address of the <see cref="EntryTable{TEntry}"/>.
+        /// </summary>
+        /// <exception cref="ObjectDisposedException"><see cref="EntryTable{TEntry}"/> instance was disposed</exception>
+        public IntPtr Base
+        {
+            get
+            {
+                if (_disposed)
+                {
+                    throw new ObjectDisposedException(null);
+                }
+
+                lock (_pages)
+                {
+                    return (IntPtr)GetRootPage();
+                }
+            }
+        }
+
+        /// <summary>
+        /// Constructs a new instance of the <see cref="AddressTable{TEntry}"/> class with the specified list of
+        /// <see cref="Level"/>.
+        /// </summary>
+        /// <exception cref="ArgumentNullException"><paramref name="levels"/> is null</exception>
+        /// <exception cref="ArgumentException">Length of <paramref name="levels"/> is less than 2</exception>
+        public AddressTable(Level[] levels)
+        {
+            if (levels == null)
+            {
+                throw new ArgumentNullException(nameof(levels));
+            }
+
+            if (levels.Length < 2)
+            {
+                throw new ArgumentException("Table must be at least 2 levels deep.", nameof(levels));
+            }
+
+            _pages = new List<IntPtr>(capacity: 16);
+
+            Levels = levels;
+            Mask = 0;
+
+            foreach (var level in Levels)
+            {
+                Mask |= level.Mask;
+            }
+        }
+
+        /// <summary>
+        /// Determines if the specified <paramref name="address"/> is in the range of the
+        /// <see cref="AddressTable{TEntry}"/>.
+        /// </summary>
+        /// <param name="address">Guest address</param>
+        /// <returns><see langword="true"/> if is valid; otherwise <see langword="false"/></returns>
+        public bool IsValid(ulong address)
+        {
+            return (address & ~Mask) == 0;
+        }
+
+        /// <summary>
+        /// Gets a reference to the value at the specified guest <paramref name="address"/>.
+        /// </summary>
+        /// <param name="address">Guest address</param>
+        /// <returns>Reference to the value at the specified guest <paramref name="address"/></returns>
+        /// <exception cref="ObjectDisposedException"><see cref="EntryTable{TEntry}"/> instance was disposed</exception>
+        /// <exception cref="ArgumentException"><paramref name="address"/> is not mapped</exception>
+        public ref TEntry GetValue(ulong address)
+        {
+            if (_disposed)
+            {
+                throw new ObjectDisposedException(null);
+            }
+
+            if (!IsValid(address))
+            {
+                throw new ArgumentException($"Address 0x{address:X} is not mapped onto the table.", nameof(address));
+            }
+
+            lock (_pages)
+            {
+                return ref GetPage(address)[Levels[^1].GetValue(address)];
+            }
+        }
+
+        /// <summary>
+        /// Gets the leaf page for the specified guest <paramref name="address"/>.
+        /// </summary>
+        /// <param name="address">Guest address</param>
+        /// <returns>Leaf page for the specified guest <paramref name="address"/></returns>
+        private TEntry* GetPage(ulong address)
+        {
+            TEntry** page = GetRootPage();
+
+            for (int i = 0; i < Levels.Length - 1; i++)
+            {
+                ref Level level = ref Levels[i];
+                ref TEntry* nextPage = ref page[level.GetValue(address)];
+
+                if (nextPage == null)
+                {
+                    ref Level nextLevel = ref Levels[i + 1];
+
+                    nextPage = i == Levels.Length - 2 ?
+                        (TEntry*)Allocate(1 << nextLevel.Length, Fill, leaf: true) :
+                        (TEntry*)Allocate(1 << nextLevel.Length, IntPtr.Zero, leaf: false);
+                }
+
+                page = (TEntry**)nextPage;
+            }
+
+            return (TEntry*)page;
+        }
+
+        /// <summary>
+        /// Lazily initialize and get the root page of the <see cref="AddressTable{TEntry}"/>.
+        /// </summary>
+        /// <returns>Root page of the <see cref="AddressTable{TEntry}"/></returns>
+        private TEntry** GetRootPage()
+        {
+            if (_table == null)
+            {
+                _table = (TEntry**)Allocate(1 << Levels[0].Length, fill: IntPtr.Zero, leaf: false);
+            }
+
+            return _table;
+        }
+
+        /// <summary>
+        /// Allocates a block of memory of the specified type and length.
+        /// </summary>
+        /// <typeparam name="T">Type of elements</typeparam>
+        /// <param name="length">Number of elements</param>
+        /// <param name="fill">Fill value</param>
+        /// <param name="leaf"><see langword="true"/> if leaf; otherwise <see langword=""="false"/></param>
+        /// <returns>Allocated block</returns>
+        private IntPtr Allocate<T>(int length, T fill, bool leaf) where T : unmanaged
+        {
+            var size = sizeof(T) * length;
+            var page = Marshal.AllocHGlobal(size);
+            var span = new Span<T>((void*)page, length);
+
+            span.Fill(fill);
+
+            _pages.Add(page);
+
+            AddressTableEventSource.Log.Allocated(size, leaf);
+
+            return page;
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="AddressTable{TEntry}"/> instance.
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        /// <summary>
+        /// Releases all unmanaged and optionally managed resources used by the <see cref="AddressTable{TEntry}"/>
+        /// instance.
+        /// </summary>
+        /// <param name="disposing"><see langword="true"/> to dispose managed resources also; otherwise just unmanaged resouces</param>
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!_disposed)
+            {
+                foreach (var page in _pages)
+                {
+                    Marshal.FreeHGlobal(page);
+                }
+
+                _disposed = true;
+            }
+        }
+
+        /// <summary>
+        /// Frees resources used by the <see cref="AddressTable{TEntry}"/> instance.
+        /// </summary>
+        ~AddressTable()
+        {
+            Dispose(false);
+        }
+    }
+}

ARMeilleure/Common/EntryTable.cs

@@ -168,7 +168,7 @@ namespace ARMeilleure.Common
         }
 
         /// <summary>
-        /// Releases all unmanaged and optionally managed resources used by the <see cref="EntryTable{TEntry}{T}"/>
+        /// Releases all unmanaged and optionally managed resources used by the <see cref="EntryTable{TEntry}"/>
         /// instance.
         /// </summary>
         /// <param name="disposing"><see langword="true"/> to dispose managed resources also; otherwise just unmanaged resouces</param>

ARMeilleure/Diagnostics/EventSources/AddressTableEventSource.cs

@@ -0,0 +1,51 @@
+using System.Diagnostics.Tracing;
+
+namespace ARMeilleure.Diagnostics.EventSources
+{
+    [EventSource(Name = "ARMeilleure")]
+    class AddressTableEventSource : EventSource
+    {
+        public static readonly AddressTableEventSource Log = new();
+
+        private ulong _size;
+        private ulong _leafSize;
+        private PollingCounter _sizeCounter;
+        private PollingCounter _leafSizeCounter;
+
+        public AddressTableEventSource()
+        {
+            _sizeCounter = new PollingCounter("addr-tab-alloc", this, () => _size / 1024d / 1024d)
+            {
+                DisplayName = "AddressTable Total Bytes Allocated",
+                DisplayUnits = "MB"
+            };
+
+            _leafSizeCounter = new PollingCounter("addr-tab-leaf-alloc", this, () => _leafSize / 1024d / 1024d)
+            {
+                DisplayName = "AddressTable Total Leaf Bytes Allocated",
+                DisplayUnits = "MB"
+            };
+        }
+
+        public void Allocated(int bytes, bool leaf)
+        {
+            _size += (uint)bytes;
+
+            if (leaf)
+            {
+                _leafSize += (uint)bytes;
+            }
+        }
+
+        protected override void Dispose(bool disposing)
+        {
+            _leafSizeCounter.Dispose();
+            _leafSizeCounter = null;
+
+            _sizeCounter.Dispose();
+            _sizeCounter = null;
+
+            base.Dispose(disposing);
+        }
+    }
+}

ARMeilleure/Instructions/InstEmitFlowHelper.cs

@@ -150,21 +150,70 @@ namespace ARMeilleure.Instructions
             }
             else
             {
-                EmitJumpTableBranch(context, Const(immediate), isJump: false);
+                EmitTableBranch(context, Const(immediate), isJump: false);
             }
         }
 
-        private static void EmitNativeCall(ArmEmitterContext context, Operand nativeContextPtr, Operand funcAddr, bool isJump)
+        public static void EmitVirtualCall(ArmEmitterContext context, Operand target)
         {
+            EmitTableBranch(context, target, isJump: false);
+        }
+
+        public static void EmitVirtualJump(ArmEmitterContext context, Operand target, bool isReturn)
+        {
+            if (isReturn)
+            {
+                context.Return(target);
+            }
+            else
+            {
+                EmitTableBranch(context, target, isJump: true);
+            }
+        }
+
+        private static void EmitTableBranch(ArmEmitterContext context, Operand guestAddress, bool isJump)
+        {
+            context.StoreToContext();
+
+            if (guestAddress.Type == OperandType.I32)
+            {
+                guestAddress = context.ZeroExtend32(OperandType.I64, guestAddress);
+            }
+
+            // Store the target guest address into the native context. The stubs uses this address to dispatch into the
+            // next translation.
+            Operand nativeContext = context.LoadArgument(OperandType.I64, 0);
+            Operand dispAddressAddr = context.Add(nativeContext, Const((ulong)NativeContext.GetDispatchAddressOffset()));
+            context.Store(dispAddressAddr, guestAddress);
+
+            Operand hostAddress;
+
+            // If address is mapped onto the function table, we can skip the table walk. Otherwise we fallback
+            // onto the dispatch stub.
+            if (guestAddress.Kind == OperandKind.Constant && context.FunctionTable.IsValid(guestAddress.Value))
+            {
+                Operand hostAddressAddr = !context.HasPtc ?
+                    Const(ref context.FunctionTable.GetValue(guestAddress.Value)) :
+                    Const(ref context.FunctionTable.GetValue(guestAddress.Value), new Symbol(SymbolType.FunctionTable, guestAddress.Value));
+
+                hostAddress = context.Load(OperandType.I64, hostAddressAddr);
+            }
+            else
+            {
+                hostAddress = !context.HasPtc ?
+                    Const((long)context.Stubs.DispatchStub) :
+                    Const((long)context.Stubs.DispatchStub, Ptc.DispatchStubSymbol);
+            }
+
             if (isJump)
             {
-                context.Tailcall(funcAddr, nativeContextPtr);
+                context.Tailcall(hostAddress, nativeContext);
             }
             else
             {
                 OpCode op = context.CurrOp;
 
-                Operand returnAddress = context.Call(funcAddr, OperandType.I64, nativeContextPtr);
+                Operand returnAddress = context.Call(hostAddress, OperandType.I64, nativeContext);
 
                 context.LoadFromContext();
 
@@ -177,203 +226,10 @@ namespace ARMeilleure.Instructions
                 // If the return address isn't to our next instruction, we need to return so the JIT can figure out
                 // what to do.
                 Operand lblContinue = context.GetLabel(nextAddr.Value);
-
-                // We need to clear out the call flag for the return address before comparing it.
                 context.BranchIf(lblContinue, returnAddress, nextAddr, Comparison.Equal, BasicBlockFrequency.Cold);
 
                 context.Return(returnAddress);
             }
         }
-
-        private static void EmitNativeCall(ArmEmitterContext context, Operand funcAddr, bool isJump)
-        {
-            EmitNativeCall(context, context.LoadArgument(OperandType.I64, 0), funcAddr, isJump);
-        }
-
-        public static void EmitVirtualCall(ArmEmitterContext context, Operand target)
-        {
-            EmitJumpTableBranch(context, target, isJump: false);
-        }
-
-        public static void EmitVirtualJump(ArmEmitterContext context, Operand target, bool isReturn)
-        {
-            if (isReturn)
-            {
-                context.Return(target);
-            }
-            else
-            {
-                EmitJumpTableBranch(context, target, isJump: true);
-            }
-        }
-
-        public static void EmitTailContinue(ArmEmitterContext context, Operand address)
-        {
-            // Left option here as it may be useful if we need to return to managed rather than tail call in future.
-            // (eg. for debug)
-            bool useTailContinue = true;
-
-            if (useTailContinue)
-            {
-                if (context.HighCq)
-                {
-                    // If we're doing a tail continue in HighCq, reserve a space in the jump table to avoid calling back
-                    // to the translator. This will always try to get a HighCq version of our continue target as well.
-                    EmitJumpTableBranch(context, address, isJump: true);
-                }
-                else
-                {
-                    context.StoreToContext();
-
-                    Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address);
-
-                    EmitNativeCall(context, fallbackAddr, isJump: true);
-                }
-            }
-            else
-            {
-                context.Return(address);
-            }
-        }
-
-        private static void EmitNativeCallWithGuestAddress(ArmEmitterContext context, Operand funcAddr, Operand guestAddress, bool isJump)
-        {
-            Operand nativeContextPtr = context.LoadArgument(OperandType.I64, 0);
-            context.Store(context.Add(nativeContextPtr, Const((long)NativeContext.GetCallAddressOffset())), guestAddress);
-
-            EmitNativeCall(context, nativeContextPtr, funcAddr, isJump);
-        }
-
-        private static void EmitBranchFallback(ArmEmitterContext context, Operand address, bool isJump)
-        {
-            Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address);
-
-            EmitNativeCall(context, fallbackAddr, isJump);
-        }
-
-        private static void EmitDynamicTableCall(ArmEmitterContext context, Operand tableAddress, Operand address, bool isJump)
-        {
-            // Loop over elements of the dynamic table. Unrolled loop.
-
-            Operand endLabel = Label();
-            Operand fallbackLabel = Label();
-
-            void EmitTableEntry(Operand entrySkipLabel)
-            {
-                // Try to take this entry in the table if its guest address equals 0.
-                Operand gotResult = context.CompareAndSwap(tableAddress, Const(0L), address);
-
-                // Is the address ours? (either taken via CompareAndSwap (0), or what was already here)
-                context.BranchIfFalse(entrySkipLabel,
-                    context.BitwiseOr(
-                        context.ICompareEqual(gotResult, address),
-                        context.ICompareEqual(gotResult, Const(0L)))
-                );
-
-                // It's ours, so what function is it pointing to?
-                Operand targetFunctionPtr = context.Add(tableAddress, Const(8L));
-                Operand targetFunction = context.Load(OperandType.I64, targetFunctionPtr);
-
-                // Call the function.
-                // We pass in the entry address as the guest address, as the entry may need to be updated by the
-                // indirect call stub.
-                EmitNativeCallWithGuestAddress(context, targetFunction, tableAddress, isJump);
-
-                context.Branch(endLabel);
-            }
-
-            // Currently this uses a size of 1, as higher values inflate code size for no real benefit.
-            for (int i = 0; i < JumpTable.DynamicTableElems; i++)
-            {
-                if (i == JumpTable.DynamicTableElems - 1)
-                {
-                    // If this is the last entry, avoid emitting the additional label and add.
-                    EmitTableEntry(fallbackLabel);
-                }
-                else
-                {
-                    Operand nextLabel = Label();
-
-                    EmitTableEntry(nextLabel);
-
-                    context.MarkLabel(nextLabel);
-
-                    // Move to the next table entry.
-                    tableAddress = context.Add(tableAddress, Const((long)JumpTable.JumpTableStride));
-                }
-            }
-
-            context.MarkLabel(fallbackLabel);
-
-            EmitBranchFallback(context, address, isJump);
-
-            context.MarkLabel(endLabel);
-        }
-
-        private static void EmitJumpTableBranch(ArmEmitterContext context, Operand address, bool isJump)
-        {
-            if (address.Type == OperandType.I32)
-            {
-                address = context.ZeroExtend32(OperandType.I64, address);
-            }
-
-            context.StoreToContext();
-
-            // TODO: Constant folding. Indirect calls are slower in the best case and emit more code so we want to
-            // avoid them when possible.
-            bool isConst = address.Kind == OperandKind.Constant;
-            ulong constAddr = address.Value;
-
-            if (!context.HighCq)
-            {
-                // Don't emit indirect calls or jumps if we're compiling in lowCq mode. This avoids wasting space on the
-                // jump and indirect tables. Just ask the translator for the function address.
-                EmitBranchFallback(context, address, isJump);
-            }
-            else if (!isConst)
-            {
-                // Virtual branch/call - store first used addresses on a small table for fast lookup.
-                int entry = context.JumpTable.ReserveDynamicEntry(context.EntryAddress, isJump);
-
-                int jumpOffset = entry * JumpTable.JumpTableStride * JumpTable.DynamicTableElems;
-
-                Operand dynTablePtr;
-
-                if (Ptc.State == PtcState.Disabled)
-                {
-                    dynTablePtr = Const(context.JumpTable.DynamicPointer.ToInt64() + jumpOffset);
-                }
-                else
-                {
-                    dynTablePtr = Const(context.JumpTable.DynamicPointer.ToInt64(), true, Ptc.DynamicPointerIndex);
-                    dynTablePtr = context.Add(dynTablePtr, Const((long)jumpOffset));
-                }
-
-                EmitDynamicTableCall(context, dynTablePtr, address, isJump);
-            }
-            else
-            {
-                int entry = context.JumpTable.ReserveTableEntry(context.EntryAddress, constAddr, isJump);
-
-                int jumpOffset = entry * JumpTable.JumpTableStride + 8; // Offset directly to the host address.
-
-                Operand tableEntryPtr;
-
-                if (Ptc.State == PtcState.Disabled)
-                {
-                    tableEntryPtr = Const(context.JumpTable.JumpPointer.ToInt64() + jumpOffset);
-                }
-                else
-                {
-                    tableEntryPtr = Const(context.JumpTable.JumpPointer.ToInt64(), true, Ptc.JumpPointerIndex);
-                    tableEntryPtr = context.Add(tableEntryPtr, Const((long)jumpOffset));
-                }
-
-                Operand funcAddr = context.Load(OperandType.I64, tableEntryPtr);
-
-                // Call the function directly. If it's not present yet, this will call the direct call stub.
-                EmitNativeCallWithGuestAddress(context, funcAddr, address, isJump);
-            }
-        }
     }
 }

ARMeilleure/Instructions/InstEmitMemoryHelper.cs

@@ -327,9 +327,9 @@ namespace ARMeilleure.Instructions
             Operand addrRotated = size != 0 ? context.RotateRight(address, Const(size)) : address;
             Operand addrShifted = context.ShiftRightUI(addrRotated, Const(PageBits - size));
 
-            Operand pte = Ptc.State == PtcState.Disabled
+            Operand pte = !context.HasPtc
                 ? Const(context.Memory.PageTablePointer.ToInt64())
-                : Const(context.Memory.PageTablePointer.ToInt64(), true, Ptc.PageTablePointerIndex);
+                : Const(context.Memory.PageTablePointer.ToInt64(), Ptc.PageTableSymbol);
 
             Operand pteOffset = context.BitwiseAnd(addrShifted, Const(addrShifted.Type, ptLevelMask));
 
@@ -411,9 +411,9 @@ namespace ARMeilleure.Instructions
                 address = context.BitwiseAnd(address, mask);
             }
 
-            Operand baseAddr = Ptc.State == PtcState.Disabled
+            Operand baseAddr = !context.HasPtc
                 ? Const(context.Memory.PageTablePointer.ToInt64())
-                : Const(context.Memory.PageTablePointer.ToInt64(), true, Ptc.PageTablePointerIndex);
+                : Const(context.Memory.PageTablePointer.ToInt64(), Ptc.PageTableSymbol);
 
             return context.Add(baseAddr, address);
         }

ARMeilleure/Instructions/NativeInterface.cs

@@ -242,23 +242,6 @@ namespace ARMeilleure.Instructions
             return (ulong)function.FuncPtr.ToInt64();
         }
 
-        public static ulong GetIndirectFunctionAddress(ulong address, ulong entryAddress)
-        {
-            TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode);
-
-            ulong ptr = (ulong)function.FuncPtr.ToInt64();
-
-            if (function.HighCq)
-            {
-                Debug.Assert(Context.Translator.JumpTable.CheckEntryFromAddressDynamicTable((IntPtr)entryAddress));
-
-                // Rewrite the host function address in the table to point to the highCq function.
-                Marshal.WriteInt64((IntPtr)entryAddress, 8, (long)ptr);
-            }
-
-            return ptr;
-        }
-
         public static bool CheckSynchronization()
         {
             Statistics.PauseTimer();

ARMeilleure/IntermediateRepresentation/Operand.cs

@@ -1,3 +1,4 @@
+using ARMeilleure.Translation.PTC;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -12,12 +13,12 @@ namespace ARMeilleure.IntermediateRepresentation
 
         public ulong Value { get; private set; }
 
-        public bool Relocatable { get; private set; }
-        public int? PtcIndex    { get; private set; }
-
         public List<Node> Assignments { get; }
         public List<Node> Uses        { get; }
 
+        public Symbol Symbol { get; private set; }
+        public bool Relocatable => Symbol.Type != SymbolType.None;
+
         public Operand()
         {
             Assignments = new List<Node>();
@@ -34,16 +35,14 @@ namespace ARMeilleure.IntermediateRepresentation
             OperandKind kind,
             OperandType type = OperandType.None,
             ulong value = 0,
-            bool relocatable = false,
+            Symbol symbol = default)
-            int? index = null)
         {
             Kind = kind;
             Type = type;
 
             Value = value;
 
-            Relocatable = relocatable;
+            Symbol = symbol;
-            PtcIndex    = index;
 
             Assignments.Clear();
             Uses.Clear();
@@ -61,9 +60,14 @@ namespace ARMeilleure.IntermediateRepresentation
             return With(OperandKind.Constant, OperandType.I32, value);
         }
 
-        public Operand With(long value, bool relocatable = false, int? index = null)
+        public Operand With(long value)
         {
-            return With(OperandKind.Constant, OperandType.I64, (ulong)value, relocatable, index);
+            return With(OperandKind.Constant, OperandType.I64, (ulong)value);
+        }
+
+        public Operand With(long value, Symbol symbol)
+        {
+            return With(OperandKind.Constant, OperandType.I64, (ulong)value, symbol);
         }
 
         public Operand With(ulong value)

ARMeilleure/IntermediateRepresentation/OperandHelper.cs

@@ -1,4 +1,5 @@
 using ARMeilleure.Common;
+using ARMeilleure.Translation.PTC;
 using System.Runtime.CompilerServices;
 
 namespace ARMeilleure.IntermediateRepresentation
@@ -25,9 +26,14 @@ namespace ARMeilleure.IntermediateRepresentation
             return Operand().With(value);
         }
 
-        public static Operand Const(long value, bool relocatable = false, int? index = null)
+        public static Operand Const(long value)
         {
-            return Operand().With(value, relocatable, index);
+            return Operand().With(value);
+        }
+
+        public static Operand Const(long value, Symbol symbol)
+        {
+            return Operand().With(value, symbol);
         }
 
         public static Operand Const(ulong value)
@@ -35,9 +41,9 @@ namespace ARMeilleure.IntermediateRepresentation
             return Operand().With(value);
         }
 
-        public static unsafe Operand Const<T>(ref T reference, int? index = null)
+        public static unsafe Operand Const<T>(ref T reference, Symbol symbol = default)
         {
-            return Operand().With((long)Unsafe.AsPointer(ref reference), index != null, index);
+            return Operand().With((long)Unsafe.AsPointer(ref reference), symbol);
         }
 
         public static Operand ConstF(float value)

ARMeilleure/Optimizations.cs

@@ -6,6 +6,9 @@ namespace ARMeilleure
     {
         public static bool FastFP { get; set; } = true;
 
+        public static bool AllowLcqInFunctionTable  { get; set; } = true;
+        public static bool UseUnmanagedDispatchLoop { get; set; } = true;
+
         public static bool UseSseIfAvailable       { get; set; } = true;
         public static bool UseSse2IfAvailable      { get; set; } = true;
         public static bool UseSse3IfAvailable      { get; set; } = true;

ARMeilleure/State/ExecutionContext.cs

@@ -66,7 +66,11 @@ namespace ARMeilleure.State
             }
         }
 
-        public bool Running { get; private set; }
+        public bool Running
+        {
+            get => _nativeContext.GetRunning();
+            private set => _nativeContext.SetRunning(value);
+        }
 
         public event EventHandler<EventArgs>              Interrupt;
         public event EventHandler<InstExceptionEventArgs> Break;
@@ -78,7 +82,6 @@ namespace ARMeilleure.State
             _hostTickFreq = 1.0 / Stopwatch.Frequency;
 
             _tickCounter = new Stopwatch();
-
             _tickCounter.Start();
         }
 
@@ -138,6 +141,7 @@ namespace ARMeilleure.State
         public void StopRunning()
         {
             Running = false;
+
             _nativeContext.SetCounter(0);
         }
 

ARMeilleure/State/NativeContext.cs

@@ -14,10 +14,11 @@ namespace ARMeilleure.State
             public fixed uint Flags[RegisterConsts.FlagsCount];
             public fixed uint FpFlags[RegisterConsts.FpFlagsCount];
             public int Counter;
-            public ulong CallAddress;
+            public ulong DispatchAddress;
             public ulong ExclusiveAddress;
             public ulong ExclusiveValueLow;
             public ulong ExclusiveValueHigh;
+            public int Running;
         }
 
         private static NativeCtxStorage _dummyStorage = new NativeCtxStorage();
@@ -117,6 +118,9 @@ namespace ARMeilleure.State
         public int GetCounter() => GetStorage().Counter;
         public void SetCounter(int value) => GetStorage().Counter = value;
 
+        public bool GetRunning() => GetStorage().Running != 0;
+        public void SetRunning(bool value) => GetStorage().Running = value ? 1 : 0;
+
         public unsafe static int GetRegisterOffset(Register reg)
         {
             if (reg.Type == RegisterType.Integer)
@@ -162,9 +166,9 @@ namespace ARMeilleure.State
             return StorageOffset(ref _dummyStorage, ref _dummyStorage.Counter);
         }
 
-        public static int GetCallAddressOffset()
+        public static int GetDispatchAddressOffset()
         {
-            return StorageOffset(ref _dummyStorage, ref _dummyStorage.CallAddress);
+            return StorageOffset(ref _dummyStorage, ref _dummyStorage.DispatchAddress);
         }
 
         public static int GetExclusiveAddressOffset()
@@ -177,6 +181,11 @@ namespace ARMeilleure.State
             return StorageOffset(ref _dummyStorage, ref _dummyStorage.ExclusiveValueLow);
         }
 
+        public static int GetRunningOffset()
+        {
+            return StorageOffset(ref _dummyStorage, ref _dummyStorage.Running);
+        }
+
         private static int StorageOffset<T>(ref NativeCtxStorage storage, ref T target)
         {
             return (int)Unsafe.ByteOffset(ref Unsafe.As<NativeCtxStorage, T>(ref storage), ref target);

ARMeilleure/Translation/ArmEmitterContext.cs

@@ -1,12 +1,14 @@
 using ARMeilleure.Common;
 using ARMeilleure.Decoders;
+using ARMeilleure.Diagnostics;
 using ARMeilleure.Instructions;
 using ARMeilleure.IntermediateRepresentation;
 using ARMeilleure.Memory;
 using ARMeilleure.State;
-using ARMeilleure.Translation.Cache;
+using ARMeilleure.Translation.PTC;
+using System;
 using System.Collections.Generic;
-
+using System.Reflection;
 using static ARMeilleure.IntermediateRepresentation.OperandHelper;
 
 namespace ARMeilleure.Translation
@@ -41,8 +43,11 @@ namespace ARMeilleure.Translation
 
         public IMemoryManager Memory { get; }
 
-        public JumpTable JumpTable { get; }
+        public bool HasPtc { get; }
+
         public EntryTable<uint> CountTable { get; }
+        public AddressTable<ulong> FunctionTable { get; }
+        public TranslatorStubs Stubs { get; }
 
         public ulong EntryAddress { get; }
         public bool HighCq { get; }
@@ -50,15 +55,18 @@ namespace ARMeilleure.Translation
 
         public ArmEmitterContext(
             IMemoryManager memory,
-            JumpTable jumpTable,
             EntryTable<uint> countTable,
+            AddressTable<ulong> funcTable,
+            TranslatorStubs stubs,
             ulong entryAddress,
             bool highCq,
             Aarch32Mode mode)
         {
+            HasPtc = Ptc.State != PtcState.Disabled;
             Memory = memory;
-            JumpTable = jumpTable;
             CountTable = countTable;
+            FunctionTable = funcTable;
+            Stubs = stubs;
             EntryAddress = entryAddress;
             HighCq = highCq;
             Mode = mode;
@@ -66,6 +74,27 @@ namespace ARMeilleure.Translation
             _labels = new Dictionary<ulong, Operand>();
         }
 
+        public override Operand Call(MethodInfo info, params Operand[] callArgs)
+        {
+            if (!HasPtc)
+            {
+                return base.Call(info, callArgs);
+            }
+            else
+            {
+                int index = Delegates.GetDelegateIndex(info);
+                IntPtr funcPtr = Delegates.GetDelegateFuncPtrByIndex(index);
+
+                OperandType returnType = GetOperandType(info.ReturnType);
+
+                Symbol symbol = new Symbol(SymbolType.DelegateTable, (ulong)index);
+
+                Symbols.Add((ulong)funcPtr.ToInt64(), info.Name);
+
+                return Call(Const(funcPtr.ToInt64(), symbol), returnType, callArgs);
+            }
+        }
+
         public Operand GetLabel(ulong address)
         {
             if (!_labels.TryGetValue(address, out Operand label))

ARMeilleure/Translation/Cache/JitCache.cs

@@ -25,6 +25,8 @@ namespace ARMeilleure.Translation.Cache
         private static readonly object _lock = new object();
         private static bool _initialized;
 
+        public static IntPtr Base => _jitRegion.Pointer;
+
         public static void Initialize(IJitMemoryAllocator allocator)
         {
             if (_initialized) return;

ARMeilleure/Translation/Cache/JumpTable.cs

@@ -1,279 +0,0 @@
-using ARMeilleure.Diagnostics;
-using ARMeilleure.Memory;
-using ARMeilleure.Translation.PTC;
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
-
-namespace ARMeilleure.Translation.Cache
-{
-    class JumpTable : IDisposable
-    {
-        // The jump table is a block of (guestAddress, hostAddress) function mappings.
-        // Each entry corresponds to one branch in a JIT compiled function. The entries are
-        // reserved specifically for each call.
-        // The Dependants dictionary can be used to update the hostAddress for any functions that change.
-
-        public const int JumpTableStride = 16; // 8 byte guest address, 8 byte host address.
-
-        private const int JumpTableSize = 1048576;
-        private const int JumpTableByteSize = JumpTableSize * JumpTableStride;
-
-        // The dynamic table is also a block of (guestAddress, hostAddress) function mappings.
-        // The main difference is that indirect calls and jumps reserve _multiple_ entries on the table.
-        // These start out as all 0. When an indirect call is made, it tries to find the guest address on the table.
-
-        // If we get to an empty address, the guestAddress is set to the call that we want.
-
-        // If we get to a guestAddress that matches our own (or we just claimed it), the hostAddress is read.
-        // If it is non-zero, we immediately branch or call the host function.
-        // If it is 0, NativeInterface is called to find the rejited address of the call.
-        // If none is found, the hostAddress entry stays at 0. Otherwise, the new address is placed in the entry.
-
-        // If the table size is exhausted and we didn't find our desired address, we fall back to requesting
-        // the function from the JIT.
-
-        public const int DynamicTableElems = 1;
-
-        public const int DynamicTableStride = DynamicTableElems * JumpTableStride;
-
-        private const int DynamicTableSize = 1048576;
-        private const int DynamicTableByteSize = DynamicTableSize * DynamicTableStride;
-
-        public const int DynamicEntryTag = 1 << 31;
-
-        private readonly ReservedRegion _jumpRegion;
-        private readonly ReservedRegion _dynamicRegion;
-
-        public IntPtr JumpPointer => _jumpRegion.Pointer;
-        public IntPtr DynamicPointer => _dynamicRegion.Pointer;
-
-        public JumpTableEntryAllocator Table { get; }
-        public JumpTableEntryAllocator DynTable { get; }
-
-        public ConcurrentDictionary<ulong, TranslatedFunction> Targets { get; }
-        public ConcurrentDictionary<ulong, List<int>> Dependants { get; } // TODO: Attach to TranslatedFunction or a wrapper class.
-        public ConcurrentDictionary<ulong, List<int>> Owners { get; }
-
-        public JumpTable(IJitMemoryAllocator allocator)
-        {
-            _jumpRegion = new ReservedRegion(allocator, JumpTableByteSize);
-            _dynamicRegion = new ReservedRegion(allocator, DynamicTableByteSize);
-
-            Table = new JumpTableEntryAllocator();
-            DynTable = new JumpTableEntryAllocator();
-
-            Targets = new ConcurrentDictionary<ulong, TranslatedFunction>();
-            Dependants = new ConcurrentDictionary<ulong, List<int>>();
-            Owners = new ConcurrentDictionary<ulong, List<int>>();
-
-            Symbols.Add((ulong)_jumpRegion.Pointer.ToInt64(), JumpTableByteSize, JumpTableStride, "JMP_TABLE");
-            Symbols.Add((ulong)_dynamicRegion.Pointer.ToInt64(), DynamicTableByteSize, DynamicTableStride, "DYN_TABLE");
-        }
-
-        public void Initialize(PtcJumpTable ptcJumpTable, ConcurrentDictionary<ulong, TranslatedFunction> funcs)
-        {
-            foreach (ulong guestAddress in ptcJumpTable.Targets)
-            {
-                if (funcs.TryGetValue(guestAddress, out TranslatedFunction func))
-                {
-                    Targets.TryAdd(guestAddress, func);
-                }
-                else
-                {
-                    throw new KeyNotFoundException($"({nameof(guestAddress)} = 0x{guestAddress:X16})");
-                }
-            }
-
-            foreach (var kv in ptcJumpTable.Dependants)
-            {
-                Dependants.TryAdd(kv.Key, new List<int>(kv.Value));
-            }
-
-            foreach (var kv in ptcJumpTable.Owners)
-            {
-                Owners.TryAdd(kv.Key, new List<int>(kv.Value));
-            }
-        }
-
-        public void RegisterFunction(ulong address, TranslatedFunction func)
-        {
-            Targets.AddOrUpdate(address, func, (key, oldFunc) => func);
-            long funcPtr = func.FuncPtr.ToInt64();
-
-            // Update all jump table entries that target this address.
-            if (Dependants.TryGetValue(address, out List<int> myDependants))
-            {
-                lock (myDependants)
-                {
-                    foreach (int entry in myDependants)
-                    {
-                        IntPtr addr = GetEntryAddressJumpTable(entry);
-
-                        Marshal.WriteInt64(addr, 8, funcPtr);
-                    }
-                }
-            }
-        }
-
-        public int ReserveTableEntry(ulong ownerGuestAddress, ulong address, bool isJump)
-        {
-            int entry = Table.AllocateEntry();
-
-            ExpandIfNeededJumpTable(entry);
-
-            // Is the address we have already registered? If so, put the function address in the jump table.
-            // If not, it will point to the direct call stub.
-            long value = DirectCallStubs.DirectCallStub(isJump).ToInt64();
-            if (Targets.TryGetValue(address, out TranslatedFunction func))
-            {
-                value = func.FuncPtr.ToInt64();
-            }
-
-            // Make sure changes to the function at the target address update this jump table entry.
-            List<int> targetDependants = Dependants.GetOrAdd(address, (addr) => new List<int>());
-            lock (targetDependants)
-            {
-                targetDependants.Add(entry);
-            }
-
-            // Keep track of ownership for jump table entries.
-            List<int> ownerEntries = Owners.GetOrAdd(ownerGuestAddress, (addr) => new List<int>());
-            lock (ownerEntries)
-            {
-                ownerEntries.Add(entry);
-            }
-
-            IntPtr addr = GetEntryAddressJumpTable(entry);
-
-            Marshal.WriteInt64(addr, 0, (long)address);
-            Marshal.WriteInt64(addr, 8, value);
-
-            return entry;
-        }
-
-        public int ReserveDynamicEntry(ulong ownerGuestAddress, bool isJump)
-        {
-            int entry = DynTable.AllocateEntry();
-
-            ExpandIfNeededDynamicTable(entry);
-
-            // Keep track of ownership for jump table entries.
-            List<int> ownerEntries = Owners.GetOrAdd(ownerGuestAddress, (addr) => new List<int>());
-            lock (ownerEntries)
-            {
-                ownerEntries.Add(entry | DynamicEntryTag);
-            }
-
-            // Initialize all host function pointers to the indirect call stub.
-            IntPtr addr = GetEntryAddressDynamicTable(entry);
-            long stubPtr = DirectCallStubs.IndirectCallStub(isJump).ToInt64();
-
-            for (int i = 0; i < DynamicTableElems; i++)
-            {
-                Marshal.WriteInt64(addr, i * JumpTableStride + 8, stubPtr);
-            }
-
-            return entry;
-        }
-
-        // For future use.
-        public void RemoveFunctionEntries(ulong guestAddress)
-        {
-            Targets.TryRemove(guestAddress, out _);
-            Dependants.TryRemove(guestAddress, out _);
-
-            if (Owners.TryRemove(guestAddress, out List<int> entries))
-            {
-                foreach (int entry in entries)
-                {
-                    if ((entry & DynamicEntryTag) == 0)
-                    {
-                        IntPtr addr = GetEntryAddressJumpTable(entry);
-
-                        Marshal.WriteInt64(addr, 0, 0L);
-                        Marshal.WriteInt64(addr, 8, 0L);
-
-                        Table.FreeEntry(entry);
-                    }
-                    else
-                    {
-                        IntPtr addr = GetEntryAddressDynamicTable(entry & ~DynamicEntryTag);
-
-                        for (int j = 0; j < DynamicTableElems; j++)
-                        {
-                            Marshal.WriteInt64(addr + j * JumpTableStride, 0, 0L);
-                            Marshal.WriteInt64(addr + j * JumpTableStride, 8, 0L);
-                        }
-
-                        DynTable.FreeEntry(entry & ~DynamicEntryTag);
-                    }
-                }
-            }
-        }
-
-        public void ExpandIfNeededJumpTable(int entry)
-        {
-            Debug.Assert(entry >= 0);
-
-            if (entry < JumpTableSize)
-            {
-                _jumpRegion.ExpandIfNeeded((ulong)((entry + 1) * JumpTableStride));
-            }
-            else
-            {
-                throw new OutOfMemoryException("JIT Direct Jump Table exhausted.");
-            }
-        }
-
-        public void ExpandIfNeededDynamicTable(int entry)
-        {
-            Debug.Assert(entry >= 0);
-
-            if (entry < DynamicTableSize)
-            {
-                _dynamicRegion.ExpandIfNeeded((ulong)((entry + 1) * DynamicTableStride));
-            }
-            else
-            {
-                throw new OutOfMemoryException("JIT Dynamic Jump Table exhausted.");
-            }
-        }
-
-        public IntPtr GetEntryAddressJumpTable(int entry)
-        {
-            Debug.Assert(Table.EntryIsValid(entry));
-
-            return _jumpRegion.Pointer + entry * JumpTableStride;
-        }
-
-        public IntPtr GetEntryAddressDynamicTable(int entry)
-        {
-            Debug.Assert(DynTable.EntryIsValid(entry));
-
-            return _dynamicRegion.Pointer + entry * DynamicTableStride;
-        }
-
-        public bool CheckEntryFromAddressJumpTable(IntPtr entryAddress)
-        {
-            int entry = Math.DivRem((int)((ulong)entryAddress - (ulong)_jumpRegion.Pointer), JumpTableStride, out int rem);
-
-            return rem == 0 && Table.EntryIsValid(entry);
-        }
-
-        public bool CheckEntryFromAddressDynamicTable(IntPtr entryAddress)
-        {
-            int entry = Math.DivRem((int)((ulong)entryAddress - (ulong)_dynamicRegion.Pointer), DynamicTableStride, out int rem);
-
-            return rem == 0 && DynTable.EntryIsValid(entry);
-        }
-
-        public void Dispose()
-        {
-            _jumpRegion.Dispose();
-            _dynamicRegion.Dispose();
-        }
-    }
-}

ARMeilleure/Translation/Cache/JumpTableEntryAllocator.cs

@@ -1,72 +0,0 @@
-using ARMeilleure.Common;
-using System.Collections.Generic;
-using System.Diagnostics;
-
-namespace ARMeilleure.Translation.Cache
-{
-    class JumpTableEntryAllocator
-    {
-        private readonly BitMap _bitmap;
-        private int _freeHint;
-
-        public JumpTableEntryAllocator()
-        {
-            _bitmap = new BitMap();
-        }
-
-        public bool EntryIsValid(int entryIndex)
-        {
-            lock (_bitmap)
-            {
-                return _bitmap.IsSet(entryIndex);
-            }
-        }
-
-        public void SetEntry(int entryIndex)
-        {
-            lock (_bitmap)
-            {
-                _bitmap.Set(entryIndex);
-            }
-        }
-
-        public int AllocateEntry()
-        {
-            lock (_bitmap)
-            {
-                int entryIndex;
-
-                if (!_bitmap.IsSet(_freeHint))
-                {
-                    entryIndex = _freeHint;
-                }
-                else
-                {
-                    entryIndex = _bitmap.FindFirstUnset();
-                }
-
-                _freeHint = entryIndex + 1;
-
-                bool wasSet = _bitmap.Set(entryIndex);
-                Debug.Assert(wasSet);
-
-                return entryIndex;
-            }
-        }
-
-        public void FreeEntry(int entryIndex)
-        {
-            lock (_bitmap)
-            {
-                _bitmap.Clear(entryIndex);
-
-                _freeHint = entryIndex;
-            }
-        }
-
-        public IEnumerable<int> GetEntries()
-        {
-            return _bitmap;
-        }
-    }
-}

ARMeilleure/Translation/Delegates.cs

@@ -114,7 +114,6 @@ namespace ARMeilleure.Translation
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpscr))); // A32 only.
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpsr)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)));
-            SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr32))); // A32 only.
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidrEl0)));

ARMeilleure/Translation/DirectCallStubs.cs

@@ -1,125 +0,0 @@
-using ARMeilleure.Instructions;
-using ARMeilleure.IntermediateRepresentation;
-using ARMeilleure.State;
-using System;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
-
-using static ARMeilleure.IntermediateRepresentation.OperandHelper;
-
-namespace ARMeilleure.Translation
-{
-    static class DirectCallStubs
-    {
-        private delegate long GuestFunction(IntPtr nativeContextPtr);
-
-        private static IntPtr _directCallStubPtr;
-        private static IntPtr _directTailCallStubPtr;
-        private static IntPtr _indirectCallStubPtr;
-        private static IntPtr _indirectTailCallStubPtr;
-
-        private static readonly object _lock = new object();
-        private static bool _initialized;
-
-        public static void InitializeStubs()
-        {
-            if (_initialized) return;
-
-            lock (_lock)
-            {
-                if (_initialized) return;
-
-                Translator.PreparePool();
-
-                _directCallStubPtr       = Marshal.GetFunctionPointerForDelegate<GuestFunction>(GenerateDirectCallStub(false));
-                _directTailCallStubPtr   = Marshal.GetFunctionPointerForDelegate<GuestFunction>(GenerateDirectCallStub(true));
-                _indirectCallStubPtr     = Marshal.GetFunctionPointerForDelegate<GuestFunction>(GenerateIndirectCallStub(false));
-                _indirectTailCallStubPtr = Marshal.GetFunctionPointerForDelegate<GuestFunction>(GenerateIndirectCallStub(true));
-
-                Translator.ResetPool();
-
-                Translator.DisposePools();
-
-                _initialized = true;
-            }
-        }
-
-        public static IntPtr DirectCallStub(bool tailCall)
-        {
-            Debug.Assert(_initialized);
-
-            return tailCall ? _directTailCallStubPtr : _directCallStubPtr;
-        }
-
-        public static IntPtr IndirectCallStub(bool tailCall)
-        {
-            Debug.Assert(_initialized);
-
-            return tailCall ? _indirectTailCallStubPtr : _indirectCallStubPtr;
-        }
-
-        private static void EmitCall(EmitterContext context, Operand address, bool tailCall)
-        {
-            if (tailCall)
-            {
-                context.Tailcall(address, context.LoadArgument(OperandType.I64, 0));
-            }
-            else
-            {
-                context.Return(context.Call(address, OperandType.I64, context.LoadArgument(OperandType.I64, 0)));
-            }
-        }
-
-        /// <summary>
-        /// Generates a stub that is used to find function addresses. Used for direct calls when their jump table does not have the host address yet.
-        /// Takes a NativeContext like a translated guest function, and extracts the target address from the NativeContext.
-        /// When the target function is compiled in highCq, all table entries are updated to point to that function instead of this stub by the translator.
-        /// </summary>
-        private static GuestFunction GenerateDirectCallStub(bool tailCall)
-        {
-            EmitterContext context = new EmitterContext();
-
-            Operand nativeContextPtr = context.LoadArgument(OperandType.I64, 0);
-
-            Operand address = context.Load(OperandType.I64, context.Add(nativeContextPtr, Const((long)NativeContext.GetCallAddressOffset())));
-
-            Operand functionAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address);
-            EmitCall(context, functionAddr, tailCall);
-
-            ControlFlowGraph cfg = context.GetControlFlowGraph();
-
-            OperandType[] argTypes = new OperandType[] { OperandType.I64 };
-
-            return Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, CompilerOptions.HighCq);
-        }
-
-        /// <summary>
-        /// Generates a stub that is used to find function addresses and add them to an indirect table.
-        /// Used for indirect calls entries (already claimed) when their jump table does not have the host address yet.
-        /// Takes a NativeContext like a translated guest function, and extracts the target indirect table entry from the NativeContext.
-        /// If the function we find is highCq, the entry in the table is updated to point to that function rather than this stub.
-        /// </summary>
-        private static GuestFunction GenerateIndirectCallStub(bool tailCall)
-        {
-            EmitterContext context = new EmitterContext();
-
-            Operand nativeContextPtr = context.LoadArgument(OperandType.I64, 0);
-
-            Operand entryAddress = context.Load(OperandType.I64, context.Add(nativeContextPtr, Const((long)NativeContext.GetCallAddressOffset())));
-            Operand address = context.Load(OperandType.I64, entryAddress);
-
-            // We need to find the missing function. If the function is HighCq, then it replaces this stub in the indirect table.
-            // Either way, we call it afterwards.
-            Operand functionAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress)), address, entryAddress);
-
-            // Call and save the function.
-            EmitCall(context, functionAddr, tailCall);
-
-            ControlFlowGraph cfg = context.GetControlFlowGraph();
-
-            OperandType[] argTypes = new OperandType[] { OperandType.I64 };
-
-            return Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, CompilerOptions.HighCq);
-        }
-    }
-}

ARMeilleure/Translation/DispatcherFunction.cs

@@ -0,0 +1,6 @@
+using System;
+
+namespace ARMeilleure.Translation
+{
+    delegate void DispatcherFunction(IntPtr nativeContext, ulong startAddress);
+}

ARMeilleure/Translation/EmitterContext.cs

@@ -97,33 +97,18 @@ namespace ARMeilleure.Translation
             return Add(Instruction.ByteSwap, Local(op1.Type), op1);
         }
 
-        public Operand Call(MethodInfo info, params Operand[] callArgs)
+        public virtual Operand Call(MethodInfo info, params Operand[] callArgs)
         {
-            if (Ptc.State == PtcState.Disabled)
+            IntPtr funcPtr = Delegates.GetDelegateFuncPtr(info);
-            {
-                IntPtr funcPtr = Delegates.GetDelegateFuncPtr(info);
 
-                OperandType returnType = GetOperandType(info.ReturnType);
+            OperandType returnType = GetOperandType(info.ReturnType);
 
-                Symbols.Add((ulong)funcPtr.ToInt64(), info.Name);
+            Symbols.Add((ulong)funcPtr.ToInt64(), info.Name);
 
-                return Call(Const(funcPtr.ToInt64()), returnType, callArgs);
+            return Call(Const(funcPtr.ToInt64()), returnType, callArgs);
-            }
-            else
-            {
-                int index = Delegates.GetDelegateIndex(info);
-
-                IntPtr funcPtr = Delegates.GetDelegateFuncPtrByIndex(index);
-
-                OperandType returnType = GetOperandType(info.ReturnType);
-
-                Symbols.Add((ulong)funcPtr.ToInt64(), info.Name);
-
-                return Call(Const(funcPtr.ToInt64(), true, index), returnType, callArgs);
-            }
         }
 
-        private static OperandType GetOperandType(Type type)
+        protected static OperandType GetOperandType(Type type)
         {
             if (type == typeof(bool)   || type == typeof(byte)  ||
                 type == typeof(char)   || type == typeof(short) ||

ARMeilleure/Translation/PTC/Ptc.cs

@@ -28,7 +28,7 @@ namespace ARMeilleure.Translation.PTC
         private const string OuterHeaderMagicString = "PTCohd\0\0";
         private const string InnerHeaderMagicString = "PTCihd\0\0";
 
-        private const uint InternalVersion = 2289; //! To be incremented manually for each change to the ARMeilleure project.
+        private const uint InternalVersion = 2228; //! To be incremented manually for each change to the ARMeilleure project.
 
         private const string ActualDir = "0";
         private const string BackupDir = "1";
@@ -36,10 +36,9 @@ namespace ARMeilleure.Translation.PTC
         private const string TitleIdTextDefault = "0000000000000000";
         private const string DisplayVersionDefault = "0";
 
-        internal const int PageTablePointerIndex = -1; // Must be a negative value.
+        internal static readonly Symbol PageTableSymbol = new(SymbolType.Special, 1);
-        internal const int JumpPointerIndex = -2; // Must be a negative value.
+        internal static readonly Symbol CountTableSymbol = new(SymbolType.Special, 2);
-        internal const int DynamicPointerIndex = -3; // Must be a negative value.
+        internal static readonly Symbol DispatchStubSymbol = new(SymbolType.Special, 3);
-        internal const int CountTableIndex = -4; // Must be a negative value.
 
         private const byte FillingByte = 0x00;
         private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest;
@@ -59,8 +58,6 @@ namespace ARMeilleure.Translation.PTC
 
         private static bool _disposed;
 
-        internal static PtcJumpTable PtcJumpTable { get; private set; }
-
         internal static string TitleIdText { get; private set; }
         internal static string DisplayVersion { get; private set; }
 
@@ -89,8 +86,6 @@ namespace ARMeilleure.Translation.PTC
 
             _disposed = false;
 
-            PtcJumpTable = new PtcJumpTable();
-
             TitleIdText = TitleIdTextDefault;
             DisplayVersion = DisplayVersionDefault;
 
@@ -348,20 +343,8 @@ namespace ARMeilleure.Translation.PTC
                             return false;
                         }
 
-                        ReadOnlySpan<byte> ptcJumpTableBytes = new(stream.PositionPointer, innerHeader.PtcJumpTableLength);
-                        stream.Seek(innerHeader.PtcJumpTableLength, SeekOrigin.Current);
-
                         Debug.Assert(stream.Position == stream.Length);
 
-                        Hash128 ptcJumpTableHash = XXHash128.ComputeHash(ptcJumpTableBytes);
-
-                        if (innerHeader.PtcJumpTableHash != ptcJumpTableHash)
-                        {
-                            InvalidateCompressedStream(compressedStream);
-
-                            return false;
-                        }
-
                         stream.Seek((long)Unsafe.SizeOf<InnerHeader>(), SeekOrigin.Begin);
 
                         _infosStream.Write(infosBytes);
@@ -375,8 +358,6 @@ namespace ARMeilleure.Translation.PTC
                         _unwindInfosStream.Write(unwindInfosBytes);
                         stream.Seek(innerHeader.UnwindInfosLength, SeekOrigin.Current);
 
-                        PtcJumpTable = PtcJumpTable.Deserialize(stream);
-
                         Debug.Assert(stream.Position == stream.Length);
                     }
                 }
@@ -422,7 +403,6 @@ namespace ARMeilleure.Translation.PTC
             finally
             {
                 ResetCarriersIfNeeded();
-                PtcJumpTable.ClearIfNeeded();
 
                 GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
             }
@@ -442,7 +422,6 @@ namespace ARMeilleure.Translation.PTC
             innerHeader.CodesLength = _codesList.Length();
             innerHeader.RelocsLength = (int)_relocsStream.Length;
             innerHeader.UnwindInfosLength = (int)_unwindInfosStream.Length;
-            innerHeader.PtcJumpTableLength = PtcJumpTable.GetSerializeSize(PtcJumpTable);
 
             OuterHeader outerHeader = new OuterHeader();
 
@@ -459,8 +438,7 @@ namespace ARMeilleure.Translation.PTC
                 innerHeader.InfosLength +
                 innerHeader.CodesLength +
                 innerHeader.RelocsLength +
-                innerHeader.UnwindInfosLength +
+                innerHeader.UnwindInfosLength;
-                innerHeader.PtcJumpTableLength;
 
             outerHeader.SetHeaderHash();
 
@@ -486,16 +464,12 @@ namespace ARMeilleure.Translation.PTC
                     ReadOnlySpan<byte> unwindInfosBytes = new(stream.PositionPointer, innerHeader.UnwindInfosLength);
                     _unwindInfosStream.WriteTo(stream);
 
-                    ReadOnlySpan<byte> ptcJumpTableBytes = new(stream.PositionPointer, innerHeader.PtcJumpTableLength);
-                    PtcJumpTable.Serialize(stream, PtcJumpTable);
-
                     Debug.Assert(stream.Position == stream.Length);
 
                     innerHeader.InfosHash = XXHash128.ComputeHash(infosBytes);
                     innerHeader.CodesHash = XXHash128.ComputeHash(codesBytes);
                     innerHeader.RelocsHash = XXHash128.ComputeHash(relocsBytes);
                     innerHeader.UnwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes);
-                    innerHeader.PtcJumpTableHash = XXHash128.ComputeHash(ptcJumpTableBytes);
 
                     innerHeader.SetHeaderHash();
 
@@ -505,7 +479,6 @@ namespace ARMeilleure.Translation.PTC
                     translatedFuncsCount = GetEntriesCount();
 
                     ResetCarriersIfNeeded();
-                    PtcJumpTable.ClearIfNeeded();
 
                     using (FileStream compressedStream = new(fileName, FileMode.OpenOrCreate))
                     using (DeflateStream deflateStream = new(compressedStream, SaveCompressionLevel, true))
@@ -545,11 +518,7 @@ namespace ARMeilleure.Translation.PTC
             }
         }
 
-        internal static void LoadTranslations(
+        internal static void LoadTranslations(Translator translator)
-            ConcurrentDictionary<ulong, TranslatedFunction> funcs,
-            IMemoryManager memory,
-            JumpTable jumpTable,
-            EntryTable<uint> countTable)
         {
             if (AreCarriersEmpty())
             {
@@ -580,7 +549,7 @@ namespace ARMeilleure.Translation.PTC
                         continue;
                     }
 
-                    bool isEntryChanged = infoEntry.Hash != ComputeHash(memory, infoEntry.Address, infoEntry.GuestSize);
+                    bool isEntryChanged = infoEntry.Hash != ComputeHash(translator.Memory, infoEntry.Address, infoEntry.GuestSize);
 
                     if (isEntryChanged || (!infoEntry.HighCq && PtcProfiler.ProfiledFuncs.TryGetValue(infoEntry.Address, out var value) && value.HighCq))
                     {
@@ -594,8 +563,6 @@ namespace ARMeilleure.Translation.PTC
 
                         if (isEntryChanged)
                         {
-                            PtcJumpTable.Clean(infoEntry.Address);
-
                             Logger.Info?.Print(LogClass.Ptc, $"Invalidated translated function (address: 0x{infoEntry.Address:X16})");
                         }
 
@@ -610,14 +577,16 @@ namespace ARMeilleure.Translation.PTC
                     {
                         RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount);
 
-                        PatchCode(code, relocEntries, memory.PageTablePointer, jumpTable, countTable, out callCounter);
+                        PatchCode(translator, code, relocEntries, out callCounter);
                     }
 
                     UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader);
 
                     TranslatedFunction func = FastTranslate(code, callCounter, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq);
 
-                    bool isAddressUnique = funcs.TryAdd(infoEntry.Address, func);
+                    translator.RegisterFunction(infoEntry.Address, func);
+
+                    bool isAddressUnique = translator.Functions.TryAdd(infoEntry.Address, func);
 
                     Debug.Assert(isAddressUnique, $"The address 0x{infoEntry.Address:X16} is not unique.");
                 }
@@ -630,12 +599,7 @@ namespace ARMeilleure.Translation.PTC
                 throw new Exception("The length of a memory stream has changed, or its position has not reached or has exceeded its end.");
             }
 
-            jumpTable.Initialize(PtcJumpTable, funcs);
+            Logger.Info?.Print(LogClass.Ptc, $"{translator.Functions.Count} translated functions loaded");
-
-            PtcJumpTable.WriteJumpTable(jumpTable, funcs);
-            PtcJumpTable.WriteDynamicTable(jumpTable);
-
-            Logger.Info?.Print(LogClass.Ptc, $"{funcs.Count} translated functions loaded");
         }
 
         private static int GetEntriesCount()
@@ -676,56 +640,63 @@ namespace ARMeilleure.Translation.PTC
             for (int i = 0; i < relocEntriesCount; i++)
             {
                 int position = relocsReader.ReadInt32();
-                int index = relocsReader.ReadInt32();
+                SymbolType type = (SymbolType)relocsReader.ReadByte();
+                ulong value = relocsReader.ReadUInt64();
 
-                relocEntries[i] = new RelocEntry(position, index);
+                relocEntries[i] = new RelocEntry(position, new Symbol(type, value));
             }
 
             return relocEntries;
         }
 
-        private static void PatchCode(
+        private static void PatchCode(Translator translator, Span<byte> code, RelocEntry[] relocEntries, out Counter<uint> callCounter)
-            Span<byte> code,
-            RelocEntry[] relocEntries,
-            IntPtr pageTablePointer,
-            JumpTable jumpTable,
-            EntryTable<uint> countTable,
-            out Counter<uint> callCounter)
         {
             callCounter = null;
 
             foreach (RelocEntry relocEntry in relocEntries)
             {
-                ulong imm;
+                IntPtr? imm = null;
+                Symbol symbol = relocEntry.Symbol;
 
-                if (relocEntry.Index == PageTablePointerIndex)
+                if (symbol.Type == SymbolType.FunctionTable)
                 {
-                    imm = (ulong)pageTablePointer.ToInt64();
+                    ulong guestAddress = symbol.Value;
-                }
-                else if (relocEntry.Index == JumpPointerIndex)
-                {
-                    imm = (ulong)jumpTable.JumpPointer.ToInt64();
-                }
-                else if (relocEntry.Index == DynamicPointerIndex)
-                {
-                    imm = (ulong)jumpTable.DynamicPointer.ToInt64();
-                }
-                else if (relocEntry.Index == CountTableIndex)
-                {
-                    callCounter = new Counter<uint>(countTable);
 
-                    unsafe { imm = (ulong)Unsafe.AsPointer(ref callCounter.Value); }
+                    if (translator.FunctionTable.IsValid(guestAddress))
+                    {
+                        unsafe { imm = (IntPtr)Unsafe.AsPointer(ref translator.FunctionTable.GetValue(guestAddress)); }
+                    }
                 }
-                else if (Delegates.TryGetDelegateFuncPtrByIndex(relocEntry.Index, out IntPtr funcPtr))
+                else if (symbol.Type == SymbolType.DelegateTable)
                 {
-                    imm = (ulong)funcPtr.ToInt64();
+                    int index = (int)symbol.Value;
+
+                    if (Delegates.TryGetDelegateFuncPtrByIndex(index, out IntPtr funcPtr))
+                    {
+                        imm = funcPtr;
+                    }
                 }
-                else
+                else if (symbol == PageTableSymbol)
+                {
+                    imm = translator.Memory.PageTablePointer;
+                }
+                else if (symbol == CountTableSymbol)
+                {
+                    callCounter = new Counter<uint>(translator.CountTable);
+
+                    unsafe { imm = (IntPtr)Unsafe.AsPointer(ref callCounter.Value); }
+                }
+                else if (symbol == DispatchStubSymbol)
+                {
+                    imm = translator.Stubs.DispatchStub;
+                }
+
+                if (imm == null)
                 {
                     throw new Exception($"Unexpected reloc entry {relocEntry}.");
                 }
 
-                BinaryPrimitives.WriteUInt64LittleEndian(code.Slice(relocEntry.Position, 8), imm);
+                BinaryPrimitives.WriteUInt64LittleEndian(code.Slice(relocEntry.Position, 8), (ulong)imm.Value);
             }
         }
 
@@ -798,13 +769,9 @@ namespace ARMeilleure.Translation.PTC
             }
         }
 
-        internal static void MakeAndSaveTranslations(
+        internal static void MakeAndSaveTranslations(Translator translator)
-            ConcurrentDictionary<ulong, TranslatedFunction> funcs,
-            IMemoryManager memory,
-            JumpTable jumpTable,
-            EntryTable<uint> countTable)
         {
-            var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(funcs);
+            var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(translator.Functions);
 
             _translateCount = 0;
             _translateTotalCount = profiledFuncsToTranslate.Count;
@@ -814,7 +781,6 @@ namespace ARMeilleure.Translation.PTC
             if (_translateTotalCount == 0 || degreeOfParallelism == 0)
             {
                 ResetCarriersIfNeeded();
-                PtcJumpTable.ClearIfNeeded();
 
                 GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
 
@@ -844,19 +810,16 @@ namespace ARMeilleure.Translation.PTC
 
                     Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address));
 
-                    TranslatedFunction func = Translator.Translate(memory, jumpTable, countTable, address, item.funcProfile.Mode, item.funcProfile.HighCq);
+                    TranslatedFunction func = translator.Translate(address, item.funcProfile.Mode, item.funcProfile.HighCq);
 
-                    bool isAddressUnique = funcs.TryAdd(address, func);
+                    bool isAddressUnique = translator.Functions.TryAdd(address, func);
 
                     Debug.Assert(isAddressUnique, $"The address 0x{address:X16} is not unique.");
 
-                    if (func.HighCq)
-                    {
-                        jumpTable.RegisterFunction(address, func);
-                    }
-
                     Interlocked.Increment(ref _translateCount);
 
+                    translator.RegisterFunction(address, func);
+
                     if (State != PtcState.Enabled)
                     {
                         break;
@@ -888,11 +851,6 @@ namespace ARMeilleure.Translation.PTC
 
             Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}");
 
-            PtcJumpTable.Initialize(jumpTable);
-
-            PtcJumpTable.ReadJumpTable(jumpTable);
-            PtcJumpTable.ReadDynamicTable(jumpTable);
-
             Thread preSaveThread = new Thread(PreSave);
             preSaveThread.IsBackground = true;
             preSaveThread.Start();
@@ -1022,13 +980,11 @@ namespace ARMeilleure.Translation.PTC
             public long CodesLength;
             public int RelocsLength;
             public int UnwindInfosLength;
-            public int PtcJumpTableLength;
 
             public Hash128 InfosHash;
             public Hash128 CodesHash;
             public Hash128 RelocsHash;
             public Hash128 UnwindInfosHash;
-            public Hash128 PtcJumpTableHash;
 
             public Hash128 HeaderHash;
 

ARMeilleure/Translation/PTC/PtcInfo.cs

@@ -30,7 +30,8 @@ namespace ARMeilleure.Translation.PTC
         public void WriteRelocEntry(RelocEntry relocEntry)
         {
             _relocWriter.Write((int)relocEntry.Position);
-            _relocWriter.Write((int)relocEntry.Index);
+            _relocWriter.Write((byte)relocEntry.Symbol.Type);
+            _relocWriter.Write((ulong)relocEntry.Symbol.Value);
 
             RelocEntriesCount++;
         }

ARMeilleure/Translation/PTC/PtcJumpTable.cs

@@ -1,350 +0,0 @@
-using ARMeilleure.Translation.Cache;
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Runtime.InteropServices;
-
-using static ARMeilleure.Translation.PTC.PtcFormatter;
-
-namespace ARMeilleure.Translation.PTC
-{
-    class PtcJumpTable
-    {
-        [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 16*/)]
-        public struct TableEntry<TAddress>
-        {
-            public int EntryIndex;
-            public long GuestAddress;
-            public TAddress HostAddress;
-
-            public TableEntry(int entryIndex, long guestAddress, TAddress hostAddress)
-            {
-                EntryIndex = entryIndex;
-                GuestAddress = guestAddress;
-                HostAddress = hostAddress;
-            }
-        }
-
-        public enum DirectHostAddress : int
-        {
-            CallStub = 0,
-            TailCallStub = 1,
-            Host = 2
-        }
-
-        public enum IndirectHostAddress : int
-        {
-            CallStub = 0,
-            TailCallStub = 1
-        }
-
-        private readonly List<TableEntry<DirectHostAddress>> _jumpTable;
-        private readonly List<TableEntry<IndirectHostAddress>> _dynamicTable;
-
-        public List<ulong> Targets { get; }
-        public Dictionary<ulong, List<int>> Dependants { get; }
-        public Dictionary<ulong, List<int>> Owners { get; }
-
-        public PtcJumpTable()
-        {
-            _jumpTable = new List<TableEntry<DirectHostAddress>>();
-            _dynamicTable = new List<TableEntry<IndirectHostAddress>>();
-
-            Targets = new List<ulong>();
-            Dependants = new Dictionary<ulong, List<int>>();
-            Owners = new Dictionary<ulong, List<int>>();
-        }
-
-        public PtcJumpTable(
-            List<TableEntry<DirectHostAddress>> jumpTable, List<TableEntry<IndirectHostAddress>> dynamicTable,
-            List<ulong> targets, Dictionary<ulong, List<int>> dependants, Dictionary<ulong, List<int>> owners)
-        {
-            _jumpTable = jumpTable;
-            _dynamicTable = dynamicTable;
-
-            Targets = targets;
-            Dependants = dependants;
-            Owners = owners;
-        }
-
-        public static PtcJumpTable Deserialize(Stream stream)
-        {
-            var jumpTable = DeserializeList<TableEntry<DirectHostAddress>>(stream);
-            var dynamicTable = DeserializeList<TableEntry<IndirectHostAddress>>(stream);
-
-            var targets = DeserializeList<ulong>(stream);
-            var dependants = DeserializeDictionary<ulong, List<int>>(stream, (stream) => DeserializeList<int>(stream));
-            var owners = DeserializeDictionary<ulong, List<int>>(stream, (stream) => DeserializeList<int>(stream));
-
-            return new PtcJumpTable(jumpTable, dynamicTable, targets, dependants, owners);
-        }
-
-        public static int GetSerializeSize(PtcJumpTable ptcJumpTable)
-        {
-            int size = 0;
-
-            size += GetSerializeSizeList(ptcJumpTable._jumpTable);
-            size += GetSerializeSizeList(ptcJumpTable._dynamicTable);
-
-            size += GetSerializeSizeList(ptcJumpTable.Targets);
-            size += GetSerializeSizeDictionary(ptcJumpTable.Dependants, (list) => GetSerializeSizeList(list));
-            size += GetSerializeSizeDictionary(ptcJumpTable.Owners, (list) => GetSerializeSizeList(list));
-
-            return size;
-        }
-
-        public static void Serialize(Stream stream, PtcJumpTable ptcJumpTable)
-        {
-            SerializeList(stream, ptcJumpTable._jumpTable);
-            SerializeList(stream, ptcJumpTable._dynamicTable);
-
-            SerializeList(stream, ptcJumpTable.Targets);
-            SerializeDictionary(stream, ptcJumpTable.Dependants, (stream, list) => SerializeList(stream, list));
-            SerializeDictionary(stream, ptcJumpTable.Owners, (stream, list) => SerializeList(stream, list));
-        }
-
-        public void Initialize(JumpTable jumpTable)
-        {
-            Targets.Clear();
-
-            foreach (ulong guestAddress in jumpTable.Targets.Keys)
-            {
-                Targets.Add(guestAddress);
-            }
-
-            Dependants.Clear();
-
-            foreach (var kv in jumpTable.Dependants)
-            {
-                Dependants.Add(kv.Key, new List<int>(kv.Value));
-            }
-
-            Owners.Clear();
-
-            foreach (var kv in jumpTable.Owners)
-            {
-                Owners.Add(kv.Key, new List<int>(kv.Value));
-            }
-        }
-
-        public void Clean(ulong guestAddress)
-        {
-            if (Owners.TryGetValue(guestAddress, out List<int> entries))
-            {
-                foreach (int entry in entries)
-                {
-                    if ((entry & JumpTable.DynamicEntryTag) == 0)
-                    {
-                        int removed = _jumpTable.RemoveAll(tableEntry => tableEntry.EntryIndex == entry);
-
-                        Debug.Assert(removed == 1);
-                    }
-                    else
-                    {
-                        if (JumpTable.DynamicTableElems > 1)
-                        {
-                            throw new NotSupportedException();
-                        }
-
-                        int removed = _dynamicTable.RemoveAll(tableEntry => tableEntry.EntryIndex == (entry & ~JumpTable.DynamicEntryTag));
-
-                        Debug.Assert(removed == 1);
-                    }
-                }
-            }
-
-            Targets.Remove(guestAddress);
-            Dependants.Remove(guestAddress);
-            Owners.Remove(guestAddress);
-        }
-
-        public void ClearIfNeeded()
-        {
-            if (_jumpTable.Count == 0 && _dynamicTable.Count == 0 &&
-                Targets.Count == 0 && Dependants.Count == 0 && Owners.Count == 0)
-            {
-                return;
-            }
-
-            _jumpTable.Clear();
-            _jumpTable.TrimExcess();
-            _dynamicTable.Clear();
-            _dynamicTable.TrimExcess();
-
-            Targets.Clear();
-            Targets.TrimExcess();
-            Dependants.Clear();
-            Dependants.TrimExcess();
-            Owners.Clear();
-            Owners.TrimExcess();
-        }
-
-        public void WriteJumpTable(JumpTable jumpTable, ConcurrentDictionary<ulong, TranslatedFunction> funcs)
-        {
-            // Writes internal state to jump table in-memory, after PtcJumpTable was deserialized.
-
-            foreach (var tableEntry in _jumpTable)
-            {
-                long guestAddress = tableEntry.GuestAddress;
-                DirectHostAddress directHostAddress = tableEntry.HostAddress;
-
-                long hostAddress;
-
-                if (directHostAddress == DirectHostAddress.CallStub)
-                {
-                    hostAddress = DirectCallStubs.DirectCallStub(false).ToInt64();
-                }
-                else if (directHostAddress == DirectHostAddress.TailCallStub)
-                {
-                    hostAddress = DirectCallStubs.DirectCallStub(true).ToInt64();
-                }
-                else if (directHostAddress == DirectHostAddress.Host)
-                {
-                    if (funcs.TryGetValue((ulong)guestAddress, out TranslatedFunction func))
-                    {
-                        hostAddress = func.FuncPtr.ToInt64();
-                    }
-                    else
-                    {
-                        if (!PtcProfiler.ProfiledFuncs.TryGetValue((ulong)guestAddress, out var value) || !value.HighCq)
-                        {
-                            throw new KeyNotFoundException($"({nameof(guestAddress)} = 0x{(ulong)guestAddress:X16})");
-                        }
-
-                        hostAddress = 0L;
-                    }
-                }
-                else
-                {
-                    throw new InvalidOperationException(nameof(directHostAddress));
-                }
-
-                int entry = tableEntry.EntryIndex;
-
-                jumpTable.Table.SetEntry(entry);
-                jumpTable.ExpandIfNeededJumpTable(entry);
-
-                IntPtr addr = jumpTable.GetEntryAddressJumpTable(entry);
-
-                Marshal.WriteInt64(addr, 0, guestAddress);
-                Marshal.WriteInt64(addr, 8, hostAddress);
-            }
-        }
-
-        public void WriteDynamicTable(JumpTable jumpTable)
-        {
-            // Writes internal state to jump table in-memory, after PtcJumpTable was deserialized.
-
-            if (JumpTable.DynamicTableElems > 1)
-            {
-                throw new NotSupportedException();
-            }
-
-            foreach (var tableEntry in _dynamicTable)
-            {
-                long guestAddress = tableEntry.GuestAddress;
-                IndirectHostAddress indirectHostAddress = tableEntry.HostAddress;
-
-                long hostAddress;
-
-                if (indirectHostAddress == IndirectHostAddress.CallStub)
-                {
-                    hostAddress = DirectCallStubs.IndirectCallStub(false).ToInt64();
-                }
-                else if (indirectHostAddress == IndirectHostAddress.TailCallStub)
-                {
-                    hostAddress = DirectCallStubs.IndirectCallStub(true).ToInt64();
-                }
-                else
-                {
-                    throw new InvalidOperationException(nameof(indirectHostAddress));
-                }
-
-                int entry = tableEntry.EntryIndex;
-
-                jumpTable.DynTable.SetEntry(entry);
-                jumpTable.ExpandIfNeededDynamicTable(entry);
-
-                IntPtr addr = jumpTable.GetEntryAddressDynamicTable(entry);
-
-                Marshal.WriteInt64(addr, 0, guestAddress);
-                Marshal.WriteInt64(addr, 8, hostAddress);
-            }
-        }
-
-        public void ReadJumpTable(JumpTable jumpTable)
-        {
-            // Reads in-memory jump table state and store internally for PtcJumpTable serialization.
-
-            _jumpTable.Clear();
-
-            IEnumerable<int> entries = jumpTable.Table.GetEntries();
-
-            foreach (int entry in entries)
-            {
-                IntPtr addr = jumpTable.GetEntryAddressJumpTable(entry);
-
-                long guestAddress = Marshal.ReadInt64(addr, 0);
-                long hostAddress = Marshal.ReadInt64(addr, 8);
-
-                DirectHostAddress directHostAddress;
-
-                if (hostAddress == DirectCallStubs.DirectCallStub(false).ToInt64())
-                {
-                    directHostAddress = DirectHostAddress.CallStub;
-                }
-                else if (hostAddress == DirectCallStubs.DirectCallStub(true).ToInt64())
-                {
-                    directHostAddress = DirectHostAddress.TailCallStub;
-                }
-                else
-                {
-                    directHostAddress = DirectHostAddress.Host;
-                }
-
-                _jumpTable.Add(new TableEntry<DirectHostAddress>(entry, guestAddress, directHostAddress));
-            }
-        }
-
-        public void ReadDynamicTable(JumpTable jumpTable)
-        {
-            // Reads in-memory jump table state and store internally for PtcJumpTable serialization.
-
-            if (JumpTable.DynamicTableElems > 1)
-            {
-                throw new NotSupportedException();
-            }
-
-            _dynamicTable.Clear();
-
-            IEnumerable<int> entries = jumpTable.DynTable.GetEntries();
-
-            foreach (int entry in entries)
-            {
-                IntPtr addr = jumpTable.GetEntryAddressDynamicTable(entry);
-
-                long guestAddress = Marshal.ReadInt64(addr, 0);
-                long hostAddress = Marshal.ReadInt64(addr, 8);
-
-                IndirectHostAddress indirectHostAddress;
-
-                if (hostAddress == DirectCallStubs.IndirectCallStub(false).ToInt64())
-                {
-                    indirectHostAddress = IndirectHostAddress.CallStub;
-                }
-                else if (hostAddress == DirectCallStubs.IndirectCallStub(true).ToInt64())
-                {
-                    indirectHostAddress = IndirectHostAddress.TailCallStub;
-                }
-                else
-                {
-                    throw new InvalidOperationException($"({nameof(hostAddress)} = 0x{hostAddress:X16})");
-                }
-
-                _dynamicTable.Add(new TableEntry<IndirectHostAddress>(entry, guestAddress, indirectHostAddress));
-            }
-        }
-    }
-}

ARMeilleure/Translation/PTC/RelocEntry.cs

@@ -2,20 +2,20 @@ namespace ARMeilleure.Translation.PTC
 {
     struct RelocEntry
     {
-        public const int Stride = 8; // Bytes.
+        public const int Stride = 13; // Bytes.
 
         public int Position;
-        public int Index;
+        public Symbol Symbol;
 
-        public RelocEntry(int position, int index)
+        public RelocEntry(int position, Symbol symbol)
         {
             Position = position;
-            Index    = index;
+            Symbol = symbol;
         }
 
         public override string ToString()
         {
-            return $"({nameof(Position)} = {Position}, {nameof(Index)} = {Index})";
+            return $"({nameof(Position)} = {Position}, {nameof(Symbol)} = {Symbol})";
         }
     }
 }

ARMeilleure/Translation/PTC/Symbol.cs

@@ -0,0 +1,100 @@
+using System;
+
+namespace ARMeilleure.Translation.PTC
+{
+    /// <summary>
+    /// Represents a symbol.
+    /// </summary>
+    struct Symbol
+    {
+        private readonly ulong _value;
+
+        /// <summary>
+        /// Gets the <see cref="SymbolType"/> of the <see cref="Symbol"/>.
+        /// </summary>
+        public SymbolType Type { get; }
+
+        /// <summary>
+        /// Gets the value of the <see cref="Symbol"/>.
+        /// </summary>
+        /// <exception cref="InvalidOperationException"><see cref="Type"/> is <see cref="SymbolType.None"/></exception>
+        public ulong Value
+        {
+            get
+            {
+                if (Type == SymbolType.None)
+                {
+                    ThrowSymbolNone();
+                }
+
+                return _value;
+            }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Symbol"/> structure with the specified <see cref="SymbolType"/> and value.
+        /// </summary>
+        /// <param name="type">Type of symbol</param>
+        /// <param name="value">Value of symbol</param>
+        public Symbol(SymbolType type, ulong value)
+        {
+            (Type, _value) = (type, value);
+        }
+
+        /// <summary>
+        /// Determines if the specified <see cref="Symbol"/> instances are equal.
+        /// </summary>
+        /// <param name="a">First instance</param>
+        /// <param name="b">Second instance</param>
+        /// <returns><see langword="true"/> if equal; otherwise <see langword="false"/></returns>
+        public static bool operator ==(Symbol a, Symbol b)
+        {
+            return a.Equals(b);
+        }
+
+        /// <summary>
+        /// Determines if the specified <see cref="Symbol"/> instances are not equal.
+        /// </summary>
+        /// <param name="a">First instance</param>
+        /// <param name="b">Second instance</param>
+        /// <returns><see langword="true"/> if not equal; otherwise <see langword="false"/></returns>
+        /// <inheritdoc/>
+        public static bool operator !=(Symbol a, Symbol b)
+        {
+            return !(a == b);
+        }
+
+        /// <summary>
+        /// Determines if the specified <see cref="Symbol"/> is equal to this <see cref="Symbol"/> instance.
+        /// </summary>
+        /// <param name="other">Other <see cref="Symbol"/> instance</param>
+        /// <returns><see langword="true"/> if equal; otherwise <see langword="false"/></returns>
+        public bool Equals(Symbol other)
+        {
+            return other.Type == Type && other._value == _value;
+        }
+
+        /// <inheritdoc/>
+        public override bool Equals(object obj)
+        {
+            return obj is Symbol sym && Equals(sym);
+        }
+
+        /// <inheritdoc/>
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Type, _value);
+        }
+
+        /// <inheritdoc/>
+        public override string ToString()
+        {
+            return $"{Type}:{_value}";
+        }
+
+        private static void ThrowSymbolNone()
+        {
+            throw new InvalidOperationException("Symbol refers to nothing.");
+        }
+    }
+}

ARMeilleure/Translation/PTC/SymbolType.cs

@@ -0,0 +1,28 @@
+namespace ARMeilleure.Translation.PTC
+{
+    /// <summary>
+    /// Types of <see cref="Symbol"/>.
+    /// </summary>
+    enum SymbolType : byte
+    {
+        /// <summary>
+        /// Refers to nothing, i.e no symbol.
+        /// </summary>
+        None,
+
+        /// <summary>
+        /// Refers to an entry in <see cref="Delegates"/>.
+        /// </summary>
+        DelegateTable,
+
+        /// <summary>
+        /// Refers to an entry in <see cref="Translator.FunctionTable"/>.
+        /// </summary>
+        FunctionTable,
+
+        /// <summary>
+        /// Refers to a special symbol which is handled by <see cref="Ptc.PatchCode"/>.
+        /// </summary>
+        Special
+    }
+}

ARMeilleure/Translation/Translator.cs

@@ -24,12 +24,27 @@ namespace ARMeilleure.Translation
 {
     public class Translator
     {
-        private const int CountTableCapacity = 4 * 1024 * 1024;
+        private static readonly AddressTable<ulong>.Level[] Levels64Bit =
+            new AddressTable<ulong>.Level[]
+            {
+                new(31, 17),
+                new(23,  8),
+                new(15,  8),
+                new( 7,  8),
+                new( 2,  5)
+            };
+
+        private static readonly AddressTable<ulong>.Level[] Levels32Bit =
+            new AddressTable<ulong>.Level[]
+            {
+                new(31, 17),
+                new(23,  8),
+                new(15,  8),
+                new( 7,  8),
+                new( 1,  6)
+            };
 
         private readonly IJitMemoryAllocator _allocator;
-        private readonly IMemoryManager _memory;
-
-        private readonly ConcurrentDictionary<ulong, TranslatedFunction> _funcs;
         private readonly ConcurrentQueue<KeyValuePair<ulong, TranslatedFunction>> _oldFuncs;
 
         private readonly ConcurrentDictionary<ulong, object> _backgroundSet;
@@ -37,21 +52,22 @@ namespace ARMeilleure.Translation
         private readonly AutoResetEvent _backgroundTranslatorEvent;
         private readonly ReaderWriterLock _backgroundTranslatorLock;
 
-        private JumpTable _jumpTable;
+        internal ConcurrentDictionary<ulong, TranslatedFunction> Functions { get; }
-        internal JumpTable JumpTable => _jumpTable;
+        internal AddressTable<ulong> FunctionTable { get; }
         internal EntryTable<uint> CountTable { get; }
+        internal TranslatorStubs Stubs { get; }
+        internal IMemoryManager Memory { get; }
 
         private volatile int _threadCount;
 
         // FIXME: Remove this once the init logic of the emulator will be redone.
         public static readonly ManualResetEvent IsReadyForTranslation = new(false);
 
-        public Translator(IJitMemoryAllocator allocator, IMemoryManager memory)
+        public Translator(IJitMemoryAllocator allocator, IMemoryManager memory, bool for64Bits)
         {
             _allocator = allocator;
-            _memory = memory;
+            Memory = memory;
 
-            _funcs = new ConcurrentDictionary<ulong, TranslatedFunction>();
             _oldFuncs = new ConcurrentQueue<KeyValuePair<ulong, TranslatedFunction>>();
 
             _backgroundSet = new ConcurrentDictionary<ulong, object>();
@@ -59,11 +75,14 @@ namespace ARMeilleure.Translation
             _backgroundTranslatorEvent = new AutoResetEvent(false);
             _backgroundTranslatorLock = new ReaderWriterLock();
 
-            CountTable = new EntryTable<uint>();
-
             JitCache.Initialize(allocator);
 
-            DirectCallStubs.InitializeStubs();
+            CountTable = new EntryTable<uint>();
+            Functions = new ConcurrentDictionary<ulong, TranslatedFunction>();
+            FunctionTable = new AddressTable<ulong>(for64Bits ? Levels64Bit : Levels32Bit);
+            Stubs = new TranslatorStubs(this);
+
+            FunctionTable.Fill = (ulong)Stubs.SlowDispatchStub;
 
             if (memory.Type.IsHostMapped())
             {
@@ -80,27 +99,21 @@ namespace ARMeilleure.Translation
                 if (_backgroundStack.TryPop(out RejitRequest request) && 
                     _backgroundSet.TryRemove(request.Address, out _))
                 {
-                    TranslatedFunction func = Translate(
+                    TranslatedFunction func = Translate(request.Address, request.Mode, highCq: true);
-                        _memory,
-                        _jumpTable,
-                        CountTable,
-                        request.Address,
-                        request.Mode,
-                        highCq: true);
 
-                    _funcs.AddOrUpdate(request.Address, func, (key, oldFunc) =>
+                    Functions.AddOrUpdate(request.Address, func, (key, oldFunc) =>
                     {
                         EnqueueForDeletion(key, oldFunc);
                         return func;
                     });
 
-                    _jumpTable.RegisterFunction(request.Address, func);
-
                     if (PtcProfiler.Enabled)
                     {
                         PtcProfiler.UpdateEntry(request.Address, request.Mode, highCq: true);
                     }
 
+                    RegisterFunction(request.Address, func);
+
                     _backgroundTranslatorLock.ReleaseReaderLock();
                 }
                 else
@@ -120,14 +133,11 @@ namespace ARMeilleure.Translation
             {
                 IsReadyForTranslation.WaitOne();
 
-                Debug.Assert(_jumpTable == null);
-                _jumpTable = new JumpTable(_allocator);
-
                 if (Ptc.State == PtcState.Enabled)
                 {
-                    Debug.Assert(_funcs.Count == 0);
+                    Debug.Assert(Functions.Count == 0);
-                    Ptc.LoadTranslations(_funcs, _memory, _jumpTable, CountTable);
+                    Ptc.LoadTranslations(this);
-                    Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable, CountTable);
+                    Ptc.MakeAndSaveTranslations(this);
                 }
 
                 PtcProfiler.Start();
@@ -160,13 +170,20 @@ namespace ARMeilleure.Translation
 
             Statistics.InitializeTimer();
 
-            NativeInterface.RegisterThread(context, _memory, this);
+            NativeInterface.RegisterThread(context, Memory, this);
 
-            do
+            if (Optimizations.UseUnmanagedDispatchLoop)
             {
-                address = ExecuteSingle(context, address);
+                Stubs.DispatchLoop(context.NativeContextPtr, address);
+            }
+            else
+            {
+                do
+                {
+                    address = ExecuteSingle(context, address);
+                }
+                while (context.Running && address != 0);
             }
-            while (context.Running && address != 0);
 
             NativeInterface.UnregisterThread();
 
@@ -178,9 +195,8 @@ namespace ARMeilleure.Translation
 
                 DisposePools();
 
-                _jumpTable.Dispose();
+                Stubs.Dispose();
-                _jumpTable = null;
+                FunctionTable.Dispose();
-
                 CountTable.Dispose();
 
                 GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
@@ -202,40 +218,51 @@ namespace ARMeilleure.Translation
 
         internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode)
         {
-            if (!_funcs.TryGetValue(address, out TranslatedFunction func))
+            if (!Functions.TryGetValue(address, out TranslatedFunction func))
             {
-                func = Translate(_memory, _jumpTable, CountTable, address, mode, highCq: false);
+                func = Translate(address, mode, highCq: false);
 
-                TranslatedFunction getFunc = _funcs.GetOrAdd(address, func);
+                TranslatedFunction oldFunc = Functions.GetOrAdd(address, func);
 
-                if (getFunc != func)
+                if (oldFunc != func)
                 {
                     JitCache.Unmap(func.FuncPtr);
-                    func = getFunc;
+                    func = oldFunc;
                 }
 
                 if (PtcProfiler.Enabled)
                 {
                     PtcProfiler.AddEntry(address, mode, highCq: false);
                 }
+
+                RegisterFunction(address, func);
             }
 
             return func;
         }
 
-        internal static TranslatedFunction Translate(
+        internal void RegisterFunction(ulong guestAddress, TranslatedFunction func)
-            IMemoryManager memory,
-            JumpTable jumpTable,
-            EntryTable<uint> countTable,
-            ulong address,
-            ExecutionMode mode,
-            bool highCq)
         {
-            var context = new ArmEmitterContext(memory, jumpTable, countTable, address, highCq, Aarch32Mode.User);
+            if (FunctionTable.IsValid(guestAddress) && (Optimizations.AllowLcqInFunctionTable || func.HighCq))
+            {
+                Volatile.Write(ref FunctionTable.GetValue(guestAddress), (ulong)func.FuncPtr);
+            }
+        }
+
+        internal TranslatedFunction Translate(ulong address, ExecutionMode mode, bool highCq)
+        {
+            var context = new ArmEmitterContext(
+                Memory,
+                CountTable,
+                FunctionTable,
+                Stubs,
+                address,
+                highCq,
+                mode: Aarch32Mode.User);
 
             Logger.StartPass(PassName.Decoding);
 
-            Block[] blocks = Decoder.Decode(memory, address, mode, highCq, singleBlock: false);
+            Block[] blocks = Decoder.Decode(Memory, address, mode, highCq, singleBlock: false);
 
             Logger.EndPass(PassName.Decoding);
 
@@ -268,7 +295,7 @@ namespace ARMeilleure.Translation
 
             GuestFunction func;
 
-            if (Ptc.State == PtcState.Disabled)
+            if (!context.HasPtc)
             {
                 func = Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, options);
 
@@ -282,7 +309,7 @@ namespace ARMeilleure.Translation
 
                 ResetPool(highCq ? 1 : 0);
 
-                Hash128 hash = Ptc.ComputeHash(memory, address, funcSize);
+                Hash128 hash = Ptc.ComputeHash(Memory, address, funcSize);
 
                 Ptc.WriteInfoCodeRelocUnwindInfo(address, funcSize, hash, highCq, ptcInfo);
             }
@@ -360,7 +387,11 @@ namespace ARMeilleure.Translation
 
                 if (block.Exit)
                 {
-                    InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address));
+                    // Left option here as it may be useful if we need to return to managed rather than tail call in
+                    // future. (eg. for debug)
+                    bool useReturns = false;
+
+                    InstEmitFlowHelper.EmitVirtualJump(context, Const(block.Address), isReturn: useReturns);
                 }
                 else
                 {
@@ -416,7 +447,10 @@ namespace ARMeilleure.Translation
 
             Operand lblEnd = Label();
 
-            Operand address = Const(ref counter.Value, Ptc.CountTableIndex);
+            Operand address = !context.HasPtc ?
+                Const(ref counter.Value) :
+                Const(ref counter.Value, Ptc.CountTableSymbol);
+
             Operand curCount = context.Load(OperandType.I32, address);
             Operand count = context.Add(curCount, Const(1));
             context.Store(address, count);
@@ -477,14 +511,14 @@ namespace ARMeilleure.Translation
             // Ensure no attempt will be made to compile new functions due to rejit.
             ClearRejitQueue(allowRequeue: false);
 
-            foreach (var func in _funcs.Values)
+            foreach (var func in Functions.Values)
             {
                 JitCache.Unmap(func.FuncPtr);
 
                 func.CallCounter?.Dispose();
             }
 
-            _funcs.Clear();
+            Functions.Clear();
 
             while (_oldFuncs.TryDequeue(out var kv))
             {
@@ -502,7 +536,7 @@ namespace ARMeilleure.Translation
             {
                 while (_backgroundStack.TryPop(out var request))
                 {
-                    if (_funcs.TryGetValue(request.Address, out var func) && func.CallCounter != null)
+                    if (Functions.TryGetValue(request.Address, out var func) && func.CallCounter != null)
                     {
                         Volatile.Write(ref func.CallCounter.Value, 0);
                     }

ARMeilleure/Translation/TranslatorStubs.cs

@@ -0,0 +1,248 @@
+using ARMeilleure.Instructions;
+using ARMeilleure.IntermediateRepresentation;
+using ARMeilleure.State;
+using ARMeilleure.Translation.Cache;
+using System;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using static ARMeilleure.IntermediateRepresentation.OperandHelper;
+
+namespace ARMeilleure.Translation
+{
+    /// <summary>
+    /// Represents a stub manager.
+    /// </summary>
+    class TranslatorStubs : IDisposable
+    {
+        private static readonly Lazy<IntPtr> _slowDispatchStub = new(GenerateSlowDispatchStub, isThreadSafe: true);
+
+        private bool _disposed;
+
+        private readonly Translator _translator;
+        private readonly Lazy<IntPtr> _dispatchStub;
+        private readonly Lazy<DispatcherFunction> _dispatchLoop;
+
+        /// <summary>
+        /// Gets the dispatch stub.
+        /// </summary>
+        /// <exception cref="ObjectDisposedException"><see cref="TranslatorStubs"/> instance was disposed</exception>
+        public IntPtr DispatchStub
+        {
+            get
+            {
+                if (_disposed)
+                {
+                    throw new ObjectDisposedException(null);
+                }
+
+                return _dispatchStub.Value;
+            }
+        }
+
+        /// <summary>
+        /// Gets the slow dispatch stub.
+        /// </summary>
+        /// <exception cref="ObjectDisposedException"><see cref="TranslatorStubs"/> instance was disposed</exception>
+        public IntPtr SlowDispatchStub
+        {
+            get
+            {
+                if (_disposed)
+                {
+                    throw new ObjectDisposedException(null);
+                }
+
+                return _slowDispatchStub.Value;
+            }
+        }
+
+        /// <summary>
+        /// Gets the dispatch loop function.
+        /// </summary>
+        /// <exception cref="ObjectDisposedException"><see cref="TranslatorStubs"/> instance was disposed</exception>
+        public DispatcherFunction DispatchLoop
+        {
+            get
+            {
+                if (_disposed)
+                {
+                    throw new ObjectDisposedException(null);
+                }
+
+                return _dispatchLoop.Value;
+            }
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="TranslatorStubs"/> class with the specified
+        /// <see cref="Translator"/> instance.
+        /// </summary>
+        /// <param name="translator"><see cref="Translator"/> instance to use</param>
+        /// <exception cref="ArgumentNullException"><paramref name="translator"/> is null</exception>
+        public TranslatorStubs(Translator translator)
+        {
+            _translator = translator ?? throw new ArgumentNullException(nameof(translator));
+            _dispatchStub = new(GenerateDispatchStub, isThreadSafe: true);
+            _dispatchLoop = new(GenerateDispatchLoop, isThreadSafe: true);
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="TranslatorStubs"/> instance.
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        /// <summary>
+        /// Releases all unmanaged and optionally managed resources used by the <see cref="TranslatorStubs"/> instance.
+        /// </summary>
+        /// <param name="disposing"><see langword="true"/> to dispose managed resources also; otherwise just unmanaged resouces</param>
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!_disposed)
+            {
+                if (_dispatchStub.IsValueCreated)
+                {
+                    JitCache.Unmap(_dispatchStub.Value);
+                }
+
+                if (_dispatchLoop.IsValueCreated)
+                {
+                    JitCache.Unmap(Marshal.GetFunctionPointerForDelegate(_dispatchLoop.Value));
+                }
+
+                _disposed = true;
+            }
+        }
+
+        /// <summary>
+        /// Frees resources used by the <see cref="TranslatorStubs"/> instance.
+        /// </summary>
+        ~TranslatorStubs()
+        {
+            Dispose(false);
+        }
+
+        /// <summary>
+        /// Generates a <see cref="DispatchStub"/>.
+        /// </summary>
+        /// <returns>Generated <see cref="DispatchStub"/></returns>
+        private IntPtr GenerateDispatchStub()
+        {
+            var context = new EmitterContext();
+
+            Operand lblFallback = Label();
+            Operand lblEnd = Label();
+
+            // Load the target guest address from the native context.
+            Operand nativeContext = context.LoadArgument(OperandType.I64, 0);
+            Operand guestAddress = context.Load(OperandType.I64,
+                context.Add(nativeContext, Const((ulong)NativeContext.GetDispatchAddressOffset())));
+
+            // Check if guest address is within range of the AddressTable.
+            Operand masked = context.BitwiseAnd(guestAddress, Const(~_translator.FunctionTable.Mask));
+            context.BranchIfTrue(lblFallback, masked);
+
+            Operand index = null;
+            Operand page = Const((long)_translator.FunctionTable.Base);
+
+            for (int i = 0; i < _translator.FunctionTable.Levels.Length; i++)
+            {
+                ref var level = ref _translator.FunctionTable.Levels[i];
+
+                // level.Mask is not used directly because it is more often bigger than 32-bits, so it will not
+                // be encoded as an immediate on x86's bitwise and operation.
+                Operand mask = Const(level.Mask >> level.Index);
+
+                index = context.BitwiseAnd(context.ShiftRightUI(guestAddress, Const(level.Index)), mask);
+
+                if (i < _translator.FunctionTable.Levels.Length - 1)
+                {
+                    page = context.Load(OperandType.I64, context.Add(page, context.ShiftLeft(index, Const(3))));
+                    context.BranchIfFalse(lblFallback, page);
+                }
+            }
+
+            Operand hostAddress;
+            Operand hostAddressAddr = context.Add(page, context.ShiftLeft(index, Const(3)));
+            hostAddress = context.Load(OperandType.I64, hostAddressAddr);
+            context.Tailcall(hostAddress, nativeContext);
+
+            context.MarkLabel(lblFallback);
+            hostAddress = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), guestAddress);
+            context.Tailcall(hostAddress, nativeContext);
+
+            var cfg = context.GetControlFlowGraph();
+            var retType = OperandType.I64;
+            var argTypes = new[] { OperandType.I64 };
+
+            var func = Compiler.Compile<GuestFunction>(cfg, argTypes, retType, CompilerOptions.HighCq);
+
+            return Marshal.GetFunctionPointerForDelegate(func);
+        }
+
+        /// <summary>
+        /// Generates a <see cref="SlowDispatchStub"/>.
+        /// </summary>
+        /// <returns>Generated <see cref="SlowDispatchStub"/></returns>
+        private static IntPtr GenerateSlowDispatchStub()
+        {
+            var context = new EmitterContext();
+
+            // Load the target guest address from the native context.
+            Operand nativeContext = context.LoadArgument(OperandType.I64, 0);
+            Operand guestAddress = context.Load(OperandType.I64,
+                context.Add(nativeContext, Const((ulong)NativeContext.GetDispatchAddressOffset())));
+
+            MethodInfo getFuncAddress = typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress));
+            Operand hostAddress = context.Call(getFuncAddress, guestAddress);
+            context.Tailcall(hostAddress, nativeContext);
+
+            var cfg = context.GetControlFlowGraph();
+            var retType = OperandType.I64;
+            var argTypes = new[] { OperandType.I64 };
+
+            var func = Compiler.Compile<GuestFunction>(cfg, argTypes, retType, CompilerOptions.HighCq);
+
+            return Marshal.GetFunctionPointerForDelegate(func);
+        }
+
+        /// <summary>
+        /// Generates a <see cref="DispatchLoop"/> function.
+        /// </summary>
+        /// <returns><see cref="DispatchLoop"/> function</returns>
+        private DispatcherFunction GenerateDispatchLoop()
+        {
+            var context = new EmitterContext();
+
+            Operand beginLbl = Label();
+            Operand endLbl = Label();
+
+            Operand nativeContext = context.LoadArgument(OperandType.I64, 0);
+            Operand guestAddress = context.Copy(
+                context.AllocateLocal(OperandType.I64),
+                context.LoadArgument(OperandType.I64, 1));
+
+            Operand runningAddress = context.Add(nativeContext, Const((ulong)NativeContext.GetRunningOffset()));
+            Operand dispatchAddress = context.Add(nativeContext, Const((ulong)NativeContext.GetDispatchAddressOffset()));
+
+            context.MarkLabel(beginLbl);
+            context.Store(dispatchAddress, guestAddress);
+            context.Copy(guestAddress, context.Call(Const((ulong)DispatchStub), OperandType.I64, nativeContext));
+            context.BranchIfFalse(endLbl, guestAddress);
+            context.BranchIfFalse(endLbl, context.Load(OperandType.I32, runningAddress));
+            context.Branch(beginLbl);
+
+            context.MarkLabel(endLbl);
+            context.Return();
+
+            var cfg = context.GetControlFlowGraph();
+            var retType = OperandType.None;
+            var argTypes = new[] { OperandType.I64, OperandType.I64 };
+
+            return Compiler.Compile<DispatcherFunction>(cfg, argTypes, retType, CompilerOptions.HighCq);
+        }
+    }
+}

Ryujinx.Cpu/CpuContext.cs

@@ -8,9 +8,9 @@ namespace Ryujinx.Cpu
     {
         private readonly Translator _translator;
 
-        public CpuContext(IMemoryManager memory)
+        public CpuContext(IMemoryManager memory, bool for64Bit)
         {
-            _translator = new Translator(new JitMemoryAllocator(), memory);
+            _translator = new Translator(new JitMemoryAllocator(), memory, for64Bit);
             memory.UnmapEvent += UnmapHandler;
         }
 

Ryujinx.HLE/HOS/ArmProcessContext.cs

@@ -3,7 +3,6 @@ using ARMeilleure.State;
 using Ryujinx.Cpu;
 using Ryujinx.HLE.HOS.Kernel.Process;
 using Ryujinx.Memory;
-using System;
 
 namespace Ryujinx.HLE.HOS
 {
@@ -14,7 +13,7 @@ namespace Ryujinx.HLE.HOS
 
         public IVirtualMemoryManager AddressSpace => _memoryManager;
 
-        public ArmProcessContext(T memoryManager)
+        public ArmProcessContext(T memoryManager, bool for64Bit)
         {
             if (memoryManager is IRefCounted rc)
             {
@@ -22,7 +21,7 @@ namespace Ryujinx.HLE.HOS
             }
 
             _memoryManager = memoryManager;
-            _cpuContext = new CpuContext(memoryManager);
+            _cpuContext = new CpuContext(memoryManager, for64Bit);
         }
 
         public void Execute(ExecutionContext context, ulong codeAddress)

Ryujinx.HLE/HOS/ArmProcessContextFactory.cs

@@ -9,19 +9,19 @@ namespace Ryujinx.HLE.HOS
 {
     class ArmProcessContextFactory : IProcessContextFactory
     {
-        public IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler)
+        public IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler, bool for64Bit)
         {
             MemoryManagerMode mode = context.Device.Configuration.MemoryManagerMode;
 
             switch (mode)
             {
                 case MemoryManagerMode.SoftwarePageTable:
-                    return new ArmProcessContext<MemoryManager>(new MemoryManager(addressSpaceSize, invalidAccessHandler));
+                    return new ArmProcessContext<MemoryManager>(new MemoryManager(addressSpaceSize, invalidAccessHandler), for64Bit);
 
                 case MemoryManagerMode.HostMapped:
                 case MemoryManagerMode.HostMappedUnsafe:
                     bool unsafeMode = mode == MemoryManagerMode.HostMappedUnsafe;
-                    return new ArmProcessContext<MemoryManagerHostMapped>(new MemoryManagerHostMapped(addressSpaceSize, unsafeMode, invalidAccessHandler));
+                    return new ArmProcessContext<MemoryManagerHostMapped>(new MemoryManagerHostMapped(addressSpaceSize, unsafeMode, invalidAccessHandler), for64Bit);
 
                 default:
                     throw new ArgumentOutOfRangeException();

Ryujinx.HLE/HOS/Kernel/Process/IProcessContextFactory.cs

@@ -1,10 +1,9 @@
-using Ryujinx.Cpu;
+using Ryujinx.Memory;
-using Ryujinx.Memory;
 
 namespace Ryujinx.HLE.HOS.Kernel.Process
 {
     interface IProcessContextFactory
     {
-        IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler);
+        IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler, bool for64Bit);
     }
 }

Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs

@@ -1049,7 +1049,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
                 _ => 39
             };
 
-            Context = _contextFactory.Create(KernelContext, 1UL << addrSpaceBits, InvalidAccessHandler);
+            bool for64Bit = flags.HasFlag(ProcessCreationFlags.Is64Bit);
+
+            Context = _contextFactory.Create(KernelContext, 1UL << addrSpaceBits, InvalidAccessHandler, for64Bit);
 
             // TODO: This should eventually be removed.
             // The GPU shouldn't depend on the CPU memory manager at all.

Ryujinx.HLE/HOS/Kernel/Process/ProcessContextFactory.cs

@@ -1,11 +1,10 @@
-using Ryujinx.Cpu;
+using Ryujinx.Memory;
-using Ryujinx.Memory;
 
 namespace Ryujinx.HLE.HOS.Kernel.Process
 {
     class ProcessContextFactory : IProcessContextFactory
     {
-        public IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler)
+        public IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler, bool for64Bit)
         {
             return new ProcessContext(new AddressSpaceManager(addressSpaceSize));
         }

Ryujinx.Tests/Cpu/CpuTest.cs

@@ -1,3 +1,4 @@
+using ARMeilleure;
 using ARMeilleure.State;
 using ARMeilleure.Translation;
 using NUnit.Framework;
@@ -60,7 +61,12 @@ namespace Ryujinx.Tests.Cpu
             _context = CpuContext.CreateExecutionContext();
             Translator.IsReadyForTranslation.Set();
 
-            _cpuContext = new CpuContext(_memory);
+            _cpuContext = new CpuContext(_memory, for64Bit: true);
+
+            // Prevent registering LCQ functions in the FunctionTable to avoid initializing and populating the table,
+            // which improves test durations.
+            Optimizations.AllowLcqInFunctionTable = false;
+            Optimizations.UseUnmanagedDispatchLoop = false;
 
             if (_unicornAvailable)
             {

Ryujinx.Tests/Cpu/CpuTest32.cs

@@ -1,4 +1,5 @@
-using ARMeilleure.State;
+using ARMeilleure;
+using ARMeilleure.State;
 using ARMeilleure.Translation;
 using NUnit.Framework;
 using Ryujinx.Cpu;
@@ -56,7 +57,12 @@ namespace Ryujinx.Tests.Cpu
             _context.IsAarch32 = true;
             Translator.IsReadyForTranslation.Set();
 
-            _cpuContext = new CpuContext(_memory);
+            _cpuContext = new CpuContext(_memory, for64Bit: false);
+
+            // Prevent registering LCQ functions in the FunctionTable to avoid initializing and populating the table,
+            // which improves test durations.
+            Optimizations.AllowLcqInFunctionTable = false;
+            Optimizations.UseUnmanagedDispatchLoop = false;
 
             if (_unicornAvailable)
             {