Thread scheduler rewrite (#393)

* Started to rewrite the thread scheduler

* Add a single core-like scheduling mode, enabled by default

* Clear exclusive monitor on context switch

* Add SetThreadActivity, misc fixes

* Implement WaitForAddress and SignalToAddress svcs, misc fixes

* Misc fixes (on SetActivity and Arbiter), other tweaks

* Rebased

* Add missing null check

* Rename multicore key on config, fix UpdatePriorityInheritance

* Make scheduling data MLQs private

* nit: Ordering

ChocolArm64/AThread.cs

@@ -10,11 +10,9 @@ namespace ChocolArm64
         public AThreadState ThreadState { get; private set; }
         public AMemory      Memory      { get; private set; }
 
-        private long EntryPoint;
-
         private ATranslator Translator;
 
-        private Thread Work;
+        public Thread Work;
 
         public event EventHandler WorkFinished;
 
@@ -24,13 +22,21 @@ namespace ChocolArm64
         {
             this.Translator = Translator;
             this.Memory     = Memory;
-            this.EntryPoint = EntryPoint;
 
             ThreadState = new AThreadState();
 
             ThreadState.ExecutionMode = AExecutionMode.AArch64;
 
             ThreadState.Running = true;
+
+            Work = new Thread(delegate()
+            {
+                Translator.ExecuteSubroutine(this, EntryPoint);
+
+                Memory.RemoveMonitor(ThreadState.Core);
+
+                WorkFinished?.Invoke(this, EventArgs.Empty);
+            });
         }
 
         public bool Execute()
@@ -40,14 +46,7 @@ namespace ChocolArm64
                 return false;
             }
 
-            Work = new Thread(delegate()
-            {
-                Translator.ExecuteSubroutine(this, EntryPoint);
-
-                Memory.RemoveMonitor(ThreadState);
-
-                WorkFinished?.Invoke(this, EventArgs.Empty);
-            });
+            Work.Name = "cpu_thread_" + Work.ManagedThreadId;
 
             Work.Start();
 
@@ -59,6 +58,11 @@ namespace ChocolArm64
             ThreadState.Running = false;
         }
 
+        public void RequestInterrupt()
+        {
+            ThreadState.RequestInterrupt();
+        }
+
         public bool IsCurrentThread()
         {
             return Thread.CurrentThread == Work;

ChocolArm64/Instruction/AInstEmitMemoryEx.cs

@@ -1,5 +1,6 @@
 using ChocolArm64.Decoder;
 using ChocolArm64.Memory;
+using ChocolArm64.State;
 using ChocolArm64.Translation;
 using System;
 using System.Reflection.Emit;
@@ -170,6 +171,8 @@ namespace ChocolArm64.Instruction
             Context.EmitLdarg(ATranslatedSub.MemoryArgIdx);
             Context.EmitLdarg(ATranslatedSub.StateArgIdx);
 
+            Context.EmitCallPropGet(typeof(AThreadState), nameof(AThreadState.Core));
+
             if (Rn != -1)
             {
                 Context.EmitLdint(Rn);

ChocolArm64/Memory/AMemory.cs

@@ -41,7 +41,7 @@ namespace ChocolArm64.Memory
             }
         }
 
-        private Dictionary<AThreadState, ArmMonitor> Monitors;
+        private Dictionary<int, ArmMonitor> Monitors;
 
         private ConcurrentDictionary<long, IntPtr> ObservedPages;
 
@@ -53,7 +53,7 @@ namespace ChocolArm64.Memory
 
         public AMemory(IntPtr Ram)
         {
-            Monitors = new Dictionary<AThreadState, ArmMonitor>();
+            Monitors = new Dictionary<int, ArmMonitor>();
 
             ObservedPages = new ConcurrentDictionary<long, IntPtr>();
 
@@ -69,17 +69,17 @@ namespace ChocolArm64.Memory
             }
         }
 
-        public void RemoveMonitor(AThreadState State)
+        public void RemoveMonitor(int Core)
         {
             lock (Monitors)
             {
-                ClearExclusive(State);
+                ClearExclusive(Core);
 
-                Monitors.Remove(State);
+                Monitors.Remove(Core);
             }
         }
 
-        public void SetExclusive(AThreadState ThreadState, long Position)
+        public void SetExclusive(int Core, long Position)
         {
             Position &= ~ErgMask;
 
@@ -93,11 +93,11 @@ namespace ChocolArm64.Memory
                     }
                 }
 
-                if (!Monitors.TryGetValue(ThreadState, out ArmMonitor ThreadMon))
+                if (!Monitors.TryGetValue(Core, out ArmMonitor ThreadMon))
                 {
                     ThreadMon = new ArmMonitor();
 
-                    Monitors.Add(ThreadState, ThreadMon);
+                    Monitors.Add(Core, ThreadMon);
                 }
 
                 ThreadMon.Position = Position;
@@ -105,7 +105,7 @@ namespace ChocolArm64.Memory
             }
         }
 
-        public bool TestExclusive(AThreadState ThreadState, long Position)
+        public bool TestExclusive(int Core, long Position)
         {
             //Note: Any call to this method also should be followed by a
             //call to ClearExclusiveForStore if this method returns true.
@@ -113,7 +113,7 @@ namespace ChocolArm64.Memory
 
             Monitor.Enter(Monitors);
 
-            if (!Monitors.TryGetValue(ThreadState, out ArmMonitor ThreadMon))
+            if (!Monitors.TryGetValue(Core, out ArmMonitor ThreadMon))
             {
                 return false;
             }
@@ -128,9 +128,9 @@ namespace ChocolArm64.Memory
             return ExState;
         }
 
-        public void ClearExclusiveForStore(AThreadState ThreadState)
+        public void ClearExclusiveForStore(int Core)
         {
-            if (Monitors.TryGetValue(ThreadState, out ArmMonitor ThreadMon))
+            if (Monitors.TryGetValue(Core, out ArmMonitor ThreadMon))
             {
                 ThreadMon.ExState = false;
             }
@@ -138,11 +138,11 @@ namespace ChocolArm64.Memory
             Monitor.Exit(Monitors);
         }
 
-        public void ClearExclusive(AThreadState ThreadState)
+        public void ClearExclusive(int Core)
         {
             lock (Monitors)
             {
-                if (Monitors.TryGetValue(ThreadState, out ArmMonitor ThreadMon))
+                if (Monitors.TryGetValue(Core, out ArmMonitor ThreadMon))
                 {
                     ThreadMon.ExState = false;
                 }

ChocolArm64/State/AThreadState.cs

@@ -41,6 +41,9 @@ namespace ChocolArm64.State
         public bool Negative;
 
         public bool Running { get; set; }
+        public int  Core    { get; set; }
+
+        private bool Interrupted;
 
         public long TpidrEl0 { get; set; }
         public long Tpidr    { get; set; }
@@ -73,6 +76,7 @@ namespace ChocolArm64.State
             }
         }
 
+        public event EventHandler<EventArgs>               Interrupt;
         public event EventHandler<AInstExceptionEventArgs> Break;
         public event EventHandler<AInstExceptionEventArgs> SvcCall;
         public event EventHandler<AInstUndefinedEventArgs> Undefined;
@@ -99,9 +103,26 @@ namespace ChocolArm64.State
 
         internal bool Synchronize()
         {
+            if (Interrupted)
+            {
+                Interrupted = false;
+
+                OnInterrupt();
+            }
+
             return Running;
         }
 
+        internal void RequestInterrupt()
+        {
+            Interrupted = true;
+        }
+
+        private void OnInterrupt()
+        {
+            Interrupt?.Invoke(this, EventArgs.Empty);
+        }
+
         internal void OnBreak(long Position, int Imm)
         {
             Break?.Invoke(this, new AInstExceptionEventArgs(Position, Imm));

Ryujinx.HLE/HOS/Horizon.cs

@@ -7,6 +7,7 @@ using Ryujinx.HLE.Loaders.Npdm;
 using Ryujinx.HLE.Logging;
 using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 
@@ -19,12 +20,22 @@ namespace Ryujinx.HLE.HOS
 
         private Switch Device;
 
-        private KProcessScheduler Scheduler;
-
         private ConcurrentDictionary<int, Process> Processes;
 
         public SystemStateMgr State { get; private set; }
 
+        internal KRecursiveLock CriticalSectionLock { get; private set; }
+
+        internal KScheduler Scheduler { get; private set; }
+
+        internal KTimeManager TimeManager { get; private set; }
+
+        internal KAddressArbiter AddressArbiter { get; private set; }
+
+        internal KSynchronization Synchronization { get; private set; }
+
+        internal LinkedList<KThread> Withholders { get; private set; }
+
         internal KSharedMemory HidSharedMem  { get; private set; }
         internal KSharedMemory FontSharedMem { get; private set; }
 
@@ -34,16 +45,28 @@ namespace Ryujinx.HLE.HOS
 
         internal Keyset KeySet { get; private set; }
 
+        private bool HasStarted;
+
         public Horizon(Switch Device)
         {
             this.Device = Device;
 
-            Scheduler = new KProcessScheduler(Device.Log);
-
             Processes = new ConcurrentDictionary<int, Process>();
 
             State = new SystemStateMgr();
 
+            CriticalSectionLock = new KRecursiveLock(this);
+
+            Scheduler = new KScheduler(this);
+
+            TimeManager = new KTimeManager();
+
+            AddressArbiter = new KAddressArbiter(this);
+
+            Synchronization = new KSynchronization(this);
+
+            Withholders = new LinkedList<KThread>();
+
             if (!Device.Memory.Allocator.TryAllocate(HidSize,  out long HidPA) ||
                 !Device.Memory.Allocator.TryAllocate(FontSize, out long FontPA))
             {
@@ -55,7 +78,7 @@ namespace Ryujinx.HLE.HOS
 
             Font = new SharedFontManager(Device, FontSharedMem.PA);
 
-            VsyncEvent = new KEvent();
+            VsyncEvent = new KEvent(this);
 
             LoadKeySet();
         }
@@ -371,10 +394,15 @@ namespace Ryujinx.HLE.HOS
             }
         }
 
-        public void SignalVsync() => VsyncEvent.WaitEvent.Set();
+        public void SignalVsync()
+        {
+            VsyncEvent.Signal();
+        }
 
         private Process MakeProcess(Npdm MetaData = null)
         {
+            HasStarted = true;
+
             Process Process;
 
             lock (Processes)
@@ -386,7 +414,7 @@ namespace Ryujinx.HLE.HOS
                     ProcessId++;
                 }
 
-                Process = new Process(Device, Scheduler, ProcessId, MetaData);
+                Process = new Process(Device, ProcessId, MetaData);
 
                 Processes.TryAdd(ProcessId, Process);
             }
@@ -409,18 +437,29 @@ namespace Ryujinx.HLE.HOS
 
                 if (Processes.Count == 0)
                 {
-                    Unload();
+                    Scheduler.Dispose();
+
+                    TimeManager.Dispose();
 
                     Device.Unload();
                 }
             }
         }
 
-        private void Unload()
+        public void EnableMultiCoreScheduling()
         {
-            VsyncEvent.Dispose();
+            if (!HasStarted)
+            {
+                Scheduler.MultiCoreScheduling = true;
+            }
+        }
 
-            Scheduler.Dispose();
+        public void DisableMultiCoreScheduling()
+        {
+            if (!HasStarted)
+            {
+                Scheduler.MultiCoreScheduling = false;
+            }
         }
 
         public void Dispose()

Ryujinx.HLE/HOS/Kernel/AddressArbiter.cs

@@ -1,111 +0,0 @@
-using ChocolArm64.Memory;
-using ChocolArm64.State;
-
-using static Ryujinx.HLE.HOS.ErrorCode;
-
-namespace Ryujinx.HLE.HOS.Kernel
-{
-    static class AddressArbiter
-    {
-        static ulong WaitForAddress(Process Process, AThreadState ThreadState, long Address, ulong Timeout)
-        {
-            KThread CurrentThread = Process.GetThread(ThreadState.Tpidr);
-
-            Process.Scheduler.SetReschedule(CurrentThread.ProcessorId);
-
-            CurrentThread.ArbiterWaitAddress = Address;
-            CurrentThread.ArbiterSignaled    = false;
-
-            Process.Scheduler.EnterWait(CurrentThread, NsTimeConverter.GetTimeMs(Timeout));
-
-            if (!CurrentThread.ArbiterSignaled)
-            {
-                return MakeError(ErrorModule.Kernel, KernelErr.Timeout);
-            }
-
-            return 0;
-        }
-
-        public static ulong WaitForAddressIfLessThan(Process      Process,
-                                                     AThreadState ThreadState,
-                                                     AMemory      Memory,
-                                                     long         Address,
-                                                     int          Value,
-                                                     ulong        Timeout,
-                                                     bool         ShouldDecrement)
-        {
-            Memory.SetExclusive(ThreadState, Address);
-
-            int CurrentValue = Memory.ReadInt32(Address);
-
-            while (true)
-            {
-                if (Memory.TestExclusive(ThreadState, Address))
-                {
-                    if (CurrentValue < Value)
-                    {
-                        if (ShouldDecrement)
-                        {
-                            Memory.WriteInt32(Address, CurrentValue - 1);
-                        }
-
-                        Memory.ClearExclusiveForStore(ThreadState);
-                    }
-                    else
-                    {
-                        Memory.ClearExclusiveForStore(ThreadState);
-
-                        return MakeError(ErrorModule.Kernel, KernelErr.InvalidState);
-                    }
-
-                    break;
-                }
-
-                Memory.SetExclusive(ThreadState, Address);
-
-                CurrentValue = Memory.ReadInt32(Address);
-            }
-
-            if (Timeout == 0)
-            {
-                return MakeError(ErrorModule.Kernel, KernelErr.Timeout);
-            }
-
-            return WaitForAddress(Process, ThreadState, Address, Timeout);
-        }
-
-        public static ulong WaitForAddressIfEqual(Process      Process,
-                                                  AThreadState ThreadState,
-                                                  AMemory      Memory,
-                                                  long         Address,
-                                                  int          Value,
-                                                  ulong        Timeout)
-        {
-            if (Memory.ReadInt32(Address) != Value)
-            {
-                return MakeError(ErrorModule.Kernel, KernelErr.InvalidState);
-            }
-
-            if (Timeout == 0)
-            {
-                return MakeError(ErrorModule.Kernel, KernelErr.Timeout);
-            }
-
-            return WaitForAddress(Process, ThreadState, Address, Timeout);
-        }
-    }
-
-    enum ArbitrationType : int
-    {
-        WaitIfLessThan,
-        DecrementAndWaitIfLessThan,
-        WaitIfEqual
-    }
-
-    enum SignalType : int
-    {
-        Signal,
-        IncrementAndSignalIfEqual,
-        ModifyByWaitingCountAndSignalIfEqual
-    }
-}

Ryujinx.HLE/HOS/Kernel/ArbitrationType.cs

@@ -0,0 +1,9 @@
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    enum ArbitrationType
+    {
+        WaitIfLessThan             = 0,
+        DecrementAndWaitIfLessThan = 1,
+        WaitIfEqual                = 2
+    }
+}

Ryujinx.HLE/HOS/Kernel/HleCoreManager.cs

@@ -0,0 +1,29 @@
+using System.Collections.Concurrent;
+using System.Threading;
+
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    class HleCoreManager
+    {
+        private ConcurrentDictionary<Thread, ManualResetEvent> Threads;
+
+        public HleCoreManager()
+        {
+            Threads = new ConcurrentDictionary<Thread, ManualResetEvent>();
+        }
+
+        public ManualResetEvent GetThread(Thread Thread)
+        {
+            return Threads.GetOrAdd(Thread, (Key) => new ManualResetEvent(false));
+        }
+
+        public void RemoveThread(Thread Thread)
+        {
+            if (Threads.TryRemove(Thread, out ManualResetEvent Event))
+            {
+                Event.Set();
+                Event.Dispose();
+            }
+        }
+    }
+}

Ryujinx.HLE/HOS/Kernel/HleScheduler.cs

@@ -0,0 +1,140 @@
+using System;
+using System.Threading;
+
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    partial class KScheduler
+    {
+        private const int RoundRobinTimeQuantumMs = 10;
+
+        private int CurrentCore;
+
+        public bool MultiCoreScheduling { get; set; }
+
+        private HleCoreManager CoreManager;
+
+        private bool KeepPreempting;
+
+        public void ContextSwitch()
+        {
+            lock (CoreContexts)
+            {
+                if (MultiCoreScheduling)
+                {
+                    int SelectedCount = 0;
+
+                    for (int Core = 0; Core < KScheduler.CpuCoresCount; Core++)
+                    {
+                        KCoreContext CoreContext = CoreContexts[Core];
+
+                        if (CoreContext.ContextSwitchNeeded && (CoreContext.CurrentThread?.Context.IsCurrentThread() ?? false))
+                        {
+                            CoreContext.ContextSwitch();
+                        }
+
+                        if (CoreContext.CurrentThread?.Context.IsCurrentThread() ?? false)
+                        {
+                            SelectedCount++;
+                        }
+                    }
+
+                    if (SelectedCount == 0)
+                    {
+                        CoreManager.GetThread(Thread.CurrentThread).Reset();
+                    }
+                    else if (SelectedCount == 1)
+                    {
+                        CoreManager.GetThread(Thread.CurrentThread).Set();
+                    }
+                    else
+                    {
+                        throw new InvalidOperationException("Thread scheduled in more than one core!");
+                    }
+                }
+                else
+                {
+                    KThread CurrentThread = CoreContexts[CurrentCore].CurrentThread;
+
+                    bool HasThreadExecuting = CurrentThread != null;
+
+                    if (HasThreadExecuting)
+                    {
+                        //If this is not the thread that is currently executing, we need
+                        //to request an interrupt to allow safely starting another thread.
+                        if (!CurrentThread.Context.IsCurrentThread())
+                        {
+                            CurrentThread.Context.RequestInterrupt();
+
+                            return;
+                        }
+
+                        CoreManager.GetThread(CurrentThread.Context.Work).Reset();
+                    }
+
+                    //Advance current core and try picking a thread,
+                    //keep advancing if it is null.
+                    for (int Core = 0; Core < 4; Core++)
+                    {
+                        CurrentCore = (CurrentCore + 1) % CpuCoresCount;
+
+                        KCoreContext CoreContext = CoreContexts[CurrentCore];
+
+                        CoreContext.UpdateCurrentThread();
+
+                        if (CoreContext.CurrentThread != null)
+                        {
+                            CoreContext.CurrentThread.ClearExclusive();
+
+                            CoreManager.GetThread(CoreContext.CurrentThread.Context.Work).Set();
+
+                            CoreContext.CurrentThread.Context.Execute();
+
+                            break;
+                        }
+                    }
+
+                    //If nothing was running before, then we are on a "external"
+                    //HLE thread, we don't need to wait.
+                    if (!HasThreadExecuting)
+                    {
+                        return;
+                    }
+                }
+            }
+
+            CoreManager.GetThread(Thread.CurrentThread).WaitOne();
+        }
+
+        private void PreemptCurrentThread()
+        {
+            //Preempts current thread every 10 milliseconds on a round-robin fashion,
+            //when multi core scheduling is disabled, to try ensuring that all threads
+            //gets a chance to run.
+            while (KeepPreempting)
+            {
+                lock (CoreContexts)
+                {
+                    KThread CurrentThread = CoreContexts[CurrentCore].CurrentThread;
+
+                    CurrentThread?.Context.RequestInterrupt();
+                }
+
+                PreemptThreads();
+
+                Thread.Sleep(RoundRobinTimeQuantumMs);
+            }
+        }
+
+        public void StopThread(KThread Thread)
+        {
+            Thread.Context.StopExecution();
+
+            CoreManager.GetThread(Thread.Context.Work).Set();
+        }
+
+        public void RemoveThread(KThread Thread)
+        {
+            CoreManager.RemoveThread(Thread.Context.Work);
+        }
+    }
+}

Ryujinx.HLE/HOS/Kernel/IKFutureSchedulerObject.cs

@@ -0,0 +1,7 @@
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    interface IKFutureSchedulerObject
+    {
+        void TimeUp();
+    }
+}

Ryujinx.HLE/HOS/Kernel/KAddressArbiter.cs

@@ -0,0 +1,678 @@
+using ChocolArm64.Memory;
+using System.Collections.Generic;
+using System.Linq;
+
+using static Ryujinx.HLE.HOS.ErrorCode;
+
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    class KAddressArbiter
+    {
+        private const int HasListenersMask = 0x40000000;
+
+        private Horizon System;
+
+        public List<KThread> CondVarThreads;
+        public List<KThread> ArbiterThreads;
+
+        public KAddressArbiter(Horizon System)
+        {
+            this.System = System;
+
+            CondVarThreads = new List<KThread>();
+            ArbiterThreads = new List<KThread>();
+        }
+
+        public long ArbitrateLock(
+            Process Process,
+            AMemory Memory,
+            int     OwnerHandle,
+            long    MutexAddress,
+            int     RequesterHandle)
+        {
+            System.CriticalSectionLock.Lock();
+
+            KThread CurrentThread = System.Scheduler.GetCurrentThread();
+
+            CurrentThread.SignaledObj   = null;
+            CurrentThread.ObjSyncResult = 0;
+
+            if (!UserToKernelInt32(Memory, MutexAddress, out int MutexValue))
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm);;
+            }
+
+            if (MutexValue != (OwnerHandle | HasListenersMask))
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return 0;
+            }
+
+            KThread MutexOwner = Process.HandleTable.GetData<KThread>(OwnerHandle);
+
+            if (MutexOwner == null)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+            }
+
+            CurrentThread.MutexAddress             = MutexAddress;
+            CurrentThread.ThreadHandleForUserMutex = RequesterHandle;
+
+            MutexOwner.AddMutexWaiter(CurrentThread);
+
+            CurrentThread.Reschedule(ThreadSchedState.Paused);
+
+            System.CriticalSectionLock.Unlock();
+            System.CriticalSectionLock.Lock();
+
+            if (CurrentThread.MutexOwner != null)
+            {
+                CurrentThread.MutexOwner.RemoveMutexWaiter(CurrentThread);
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            return (uint)CurrentThread.ObjSyncResult;
+        }
+
+        public long ArbitrateUnlock(AMemory Memory, long MutexAddress)
+        {
+            System.CriticalSectionLock.Lock();
+
+            KThread CurrentThread = System.Scheduler.GetCurrentThread();
+
+            (long Result, KThread NewOwnerThread) = MutexUnlock(Memory, CurrentThread, MutexAddress);
+
+            if (Result != 0 && NewOwnerThread != null)
+            {
+                NewOwnerThread.SignaledObj   = null;
+                NewOwnerThread.ObjSyncResult = (int)Result;
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            return Result;
+        }
+
+        public long WaitProcessWideKeyAtomic(
+            AMemory Memory,
+            long    MutexAddress,
+            long    CondVarAddress,
+            int     ThreadHandle,
+            long    Timeout)
+        {
+            System.CriticalSectionLock.Lock();
+
+            KThread CurrentThread = System.Scheduler.GetCurrentThread();
+
+            CurrentThread.SignaledObj   = null;
+            CurrentThread.ObjSyncResult = (int)MakeError(ErrorModule.Kernel, KernelErr.Timeout);
+
+            if (CurrentThread.ShallBeTerminated ||
+                CurrentThread.SchedFlags == ThreadSchedState.TerminationPending)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.ThreadTerminating);
+            }
+
+            (long Result, _) = MutexUnlock(Memory, CurrentThread, MutexAddress);
+
+            if (Result != 0)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return Result;
+            }
+
+            CurrentThread.MutexAddress             = MutexAddress;
+            CurrentThread.ThreadHandleForUserMutex = ThreadHandle;
+            CurrentThread.CondVarAddress           = CondVarAddress;
+
+            CondVarThreads.Add(CurrentThread);
+
+            if (Timeout != 0)
+            {
+                CurrentThread.Reschedule(ThreadSchedState.Paused);
+
+                if (Timeout > 0)
+                {
+                    System.TimeManager.ScheduleFutureInvocation(CurrentThread, Timeout);
+                }
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            if (Timeout > 0)
+            {
+                System.TimeManager.UnscheduleFutureInvocation(CurrentThread);
+            }
+
+            System.CriticalSectionLock.Lock();
+
+            if (CurrentThread.MutexOwner != null)
+            {
+                CurrentThread.MutexOwner.RemoveMutexWaiter(CurrentThread);
+            }
+
+            CondVarThreads.Remove(CurrentThread);
+
+            System.CriticalSectionLock.Unlock();
+
+            return (uint)CurrentThread.ObjSyncResult;
+        }
+
+        private (long, KThread) MutexUnlock(AMemory Memory, KThread CurrentThread, long MutexAddress)
+        {
+            KThread NewOwnerThread = CurrentThread.RelinquishMutex(MutexAddress, out int Count);
+
+            int MutexValue = 0;
+
+            if (NewOwnerThread != null)
+            {
+                MutexValue = NewOwnerThread.ThreadHandleForUserMutex;
+
+                if (Count >= 2)
+                {
+                    MutexValue |= HasListenersMask;
+                }
+
+                NewOwnerThread.SignaledObj   = null;
+                NewOwnerThread.ObjSyncResult = 0;
+
+                NewOwnerThread.ReleaseAndResume();
+            }
+
+            long Result = 0;
+
+            if (!KernelToUserInt32(Memory, MutexAddress, MutexValue))
+            {
+                Result = MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm);
+            }
+
+            return (Result, NewOwnerThread);
+        }
+
+        public void SignalProcessWideKey(Process Process, AMemory Memory, long Address, int Count)
+        {
+            Queue<KThread> SignaledThreads = new Queue<KThread>();
+
+            System.CriticalSectionLock.Lock();
+
+            IOrderedEnumerable<KThread> SortedThreads = CondVarThreads.OrderBy(x => x.DynamicPriority);
+
+            foreach (KThread Thread in SortedThreads.Where(x => x.CondVarAddress == Address))
+            {
+                TryAcquireMutex(Process, Memory, Thread);
+
+                SignaledThreads.Enqueue(Thread);
+
+                //If the count is <= 0, we should signal all threads waiting.
+                if (Count >= 1 && --Count == 0)
+                {
+                    break;
+                }
+            }
+
+            while (SignaledThreads.TryDequeue(out KThread Thread))
+            {
+                CondVarThreads.Remove(Thread);
+            }
+
+            System.CriticalSectionLock.Unlock();
+        }
+
+        private KThread TryAcquireMutex(Process Process, AMemory Memory, KThread Requester)
+        {
+            long Address = Requester.MutexAddress;
+
+            Memory.SetExclusive(0, Address);
+
+            if (!UserToKernelInt32(Memory, Address, out int MutexValue))
+            {
+                //Invalid address.
+                Memory.ClearExclusive(0);
+
+                Requester.SignaledObj   = null;
+                Requester.ObjSyncResult = (int)MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm);
+
+                return null;
+            }
+
+            while (true)
+            {
+                if (Memory.TestExclusive(0, Address))
+                {
+                    if (MutexValue != 0)
+                    {
+                        //Update value to indicate there is a mutex waiter now.
+                        Memory.WriteInt32(Address, MutexValue | HasListenersMask);
+                    }
+                    else
+                    {
+                        //No thread owning the mutex, assign to requesting thread.
+                        Memory.WriteInt32(Address, Requester.ThreadHandleForUserMutex);
+                    }
+
+                    Memory.ClearExclusiveForStore(0);
+
+                    break;
+                }
+
+                Memory.SetExclusive(0, Address);
+
+                MutexValue = Memory.ReadInt32(Address);
+            }
+
+            if (MutexValue == 0)
+            {
+                //We now own the mutex.
+                Requester.SignaledObj   = null;
+                Requester.ObjSyncResult = 0;
+
+                Requester.ReleaseAndResume();
+
+                return null;
+            }
+
+            MutexValue &= ~HasListenersMask;
+
+            KThread MutexOwner = Process.HandleTable.GetData<KThread>(MutexValue);
+
+            if (MutexOwner != null)
+            {
+                //Mutex already belongs to another thread, wait for it.
+                MutexOwner.AddMutexWaiter(Requester);
+            }
+            else
+            {
+                //Invalid mutex owner.
+                Requester.SignaledObj   = null;
+                Requester.ObjSyncResult = (int)MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+
+                Requester.ReleaseAndResume();
+            }
+
+            return MutexOwner;
+        }
+
+        public long WaitForAddressIfEqual(AMemory Memory, long Address, int Value, long Timeout)
+        {
+            KThread CurrentThread = System.Scheduler.GetCurrentThread();
+
+            System.CriticalSectionLock.Lock();
+
+            if (CurrentThread.ShallBeTerminated ||
+                CurrentThread.SchedFlags == ThreadSchedState.TerminationPending)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.ThreadTerminating);
+            }
+
+            CurrentThread.SignaledObj   = null;
+            CurrentThread.ObjSyncResult = (int)MakeError(ErrorModule.Kernel, KernelErr.Timeout);
+
+            if (!UserToKernelInt32(Memory, Address, out int CurrentValue))
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm);
+            }
+
+            if (CurrentValue == Value)
+            {
+                if (Timeout == 0)
+                {
+                    System.CriticalSectionLock.Unlock();
+
+                    return MakeError(ErrorModule.Kernel, KernelErr.Timeout);
+                }
+
+                CurrentThread.MutexAddress         = Address;
+                CurrentThread.WaitingInArbitration = true;
+
+                InsertSortedByPriority(ArbiterThreads, CurrentThread);
+
+                CurrentThread.Reschedule(ThreadSchedState.Paused);
+
+                if (Timeout > 0)
+                {
+                    System.TimeManager.ScheduleFutureInvocation(CurrentThread, Timeout);
+                }
+
+                System.CriticalSectionLock.Unlock();
+
+                if (Timeout > 0)
+                {
+                    System.TimeManager.UnscheduleFutureInvocation(CurrentThread);
+                }
+
+                System.CriticalSectionLock.Lock();
+
+                if (CurrentThread.WaitingInArbitration)
+                {
+                    ArbiterThreads.Remove(CurrentThread);
+
+                    CurrentThread.WaitingInArbitration = false;
+                }
+
+                System.CriticalSectionLock.Unlock();
+
+                return CurrentThread.ObjSyncResult;
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            return MakeError(ErrorModule.Kernel, KernelErr.InvalidState);
+        }
+
+        public long WaitForAddressIfLessThan(
+            AMemory Memory,
+            long    Address,
+            int     Value,
+            bool    ShouldDecrement,
+            long    Timeout)
+        {
+            KThread CurrentThread = System.Scheduler.GetCurrentThread();
+
+            System.CriticalSectionLock.Lock();
+
+            if (CurrentThread.ShallBeTerminated ||
+                CurrentThread.SchedFlags == ThreadSchedState.TerminationPending)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.ThreadTerminating);
+            }
+
+            CurrentThread.SignaledObj   = null;
+            CurrentThread.ObjSyncResult = (int)MakeError(ErrorModule.Kernel, KernelErr.Timeout);
+
+            //If ShouldDecrement is true, do atomic decrement of the value at Address.
+            Memory.SetExclusive(0, Address);
+
+            if (!UserToKernelInt32(Memory, Address, out int CurrentValue))
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm);
+            }
+
+            if (ShouldDecrement)
+            {
+                while (CurrentValue < Value)
+                {
+                    if (Memory.TestExclusive(0, Address))
+                    {
+                        Memory.WriteInt32(Address, CurrentValue - 1);
+
+                        Memory.ClearExclusiveForStore(0);
+
+                        break;
+                    }
+
+                    Memory.SetExclusive(0, Address);
+
+                    CurrentValue = Memory.ReadInt32(Address);
+                }
+            }
+
+            Memory.ClearExclusive(0);
+
+            if (CurrentValue < Value)
+            {
+                if (Timeout == 0)
+                {
+                    System.CriticalSectionLock.Unlock();
+
+                    return MakeError(ErrorModule.Kernel, KernelErr.Timeout);
+                }
+
+                CurrentThread.MutexAddress         = Address;
+                CurrentThread.WaitingInArbitration = true;
+
+                InsertSortedByPriority(ArbiterThreads, CurrentThread);
+
+                CurrentThread.Reschedule(ThreadSchedState.Paused);
+
+                if (Timeout > 0)
+                {
+                    System.TimeManager.ScheduleFutureInvocation(CurrentThread, Timeout);
+                }
+
+                System.CriticalSectionLock.Unlock();
+
+                if (Timeout > 0)
+                {
+                    System.TimeManager.UnscheduleFutureInvocation(CurrentThread);
+                }
+
+                System.CriticalSectionLock.Lock();
+
+                if (CurrentThread.WaitingInArbitration)
+                {
+                    ArbiterThreads.Remove(CurrentThread);
+
+                    CurrentThread.WaitingInArbitration = false;
+                }
+
+                System.CriticalSectionLock.Unlock();
+
+                return CurrentThread.ObjSyncResult;
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            return MakeError(ErrorModule.Kernel, KernelErr.InvalidState);
+        }
+
+        private void InsertSortedByPriority(List<KThread> Threads, KThread Thread)
+        {
+            int NextIndex = -1;
+
+            for (int Index = 0; Index < Threads.Count; Index++)
+            {
+                if (Threads[Index].DynamicPriority > Thread.DynamicPriority)
+                {
+                    NextIndex = Index;
+
+                    break;
+                }
+            }
+
+            if (NextIndex != -1)
+            {
+                Threads.Insert(NextIndex, Thread);
+            }
+            else
+            {
+                Threads.Add(Thread);
+            }
+        }
+
+        public long Signal(long Address, int Count)
+        {
+            System.CriticalSectionLock.Lock();
+
+            WakeArbiterThreads(Address, Count);
+
+            System.CriticalSectionLock.Unlock();
+
+            return 0;
+        }
+
+        public long SignalAndIncrementIfEqual(AMemory Memory, long Address, int Value, int Count)
+        {
+            System.CriticalSectionLock.Lock();
+
+            Memory.SetExclusive(0, Address);
+
+            if (!UserToKernelInt32(Memory, Address, out int CurrentValue))
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm);
+            }
+
+            while (CurrentValue == Value)
+            {
+                if (Memory.TestExclusive(0, Address))
+                {
+                    Memory.WriteInt32(Address, CurrentValue + 1);
+
+                    Memory.ClearExclusiveForStore(0);
+
+                    break;
+                }
+
+                Memory.SetExclusive(0, Address);
+
+                CurrentValue = Memory.ReadInt32(Address);
+            }
+
+            Memory.ClearExclusive(0);
+
+            if (CurrentValue != Value)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.InvalidState);
+            }
+
+            WakeArbiterThreads(Address, Count);
+
+            System.CriticalSectionLock.Unlock();
+
+            return 0;
+        }
+
+        public long SignalAndModifyIfEqual(AMemory Memory, long Address, int Value, int Count)
+        {
+            System.CriticalSectionLock.Lock();
+
+            int Offset;
+
+            //The value is decremented if the number of threads waiting is less
+            //or equal to the Count of threads to be signaled, or Count is zero
+            //or negative. It is incremented if there are no threads waiting.
+            int WaitingCount = 0;
+
+            foreach (KThread Thread in ArbiterThreads.Where(x => x.MutexAddress == Address))
+            {
+                if (++WaitingCount > Count)
+                {
+                    break;
+                }
+            }
+
+            if (WaitingCount > 0)
+            {
+                Offset = WaitingCount <= Count || Count <= 0 ? -1 : 0;
+            }
+            else
+            {
+                Offset = 1;
+            }
+
+            Memory.SetExclusive(0, Address);
+
+            if (!UserToKernelInt32(Memory, Address, out int CurrentValue))
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm);
+            }
+
+            while (CurrentValue == Value)
+            {
+                if (Memory.TestExclusive(0, Address))
+                {
+                    Memory.WriteInt32(Address, CurrentValue + Offset);
+
+                    Memory.ClearExclusiveForStore(0);
+
+                    break;
+                }
+
+                Memory.SetExclusive(0, Address);
+
+                CurrentValue = Memory.ReadInt32(Address);
+            }
+
+            Memory.ClearExclusive(0);
+
+            if (CurrentValue != Value)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.InvalidState);
+            }
+
+            WakeArbiterThreads(Address, Count);
+
+            System.CriticalSectionLock.Unlock();
+
+            return 0;
+        }
+
+        private void WakeArbiterThreads(long Address, int Count)
+        {
+            Queue<KThread> SignaledThreads = new Queue<KThread>();
+
+            foreach (KThread Thread in ArbiterThreads.Where(x => x.MutexAddress == Address))
+            {
+                SignaledThreads.Enqueue(Thread);
+
+                //If the count is <= 0, we should signal all threads waiting.
+                if (Count >= 1 && --Count == 0)
+                {
+                    break;
+                }
+            }
+
+            while (SignaledThreads.TryDequeue(out KThread Thread))
+            {
+                Thread.SignaledObj   = null;
+                Thread.ObjSyncResult = 0;
+
+                Thread.ReleaseAndResume();
+
+                Thread.WaitingInArbitration = false;
+
+                ArbiterThreads.Remove(Thread);
+            }
+        }
+
+        private bool UserToKernelInt32(AMemory Memory, long Address, out int Value)
+        {
+            if (Memory.IsMapped(Address))
+            {
+                Value = Memory.ReadInt32(Address);
+
+                return true;
+            }
+
+            Value = 0;
+
+            return false;
+        }
+
+        private bool KernelToUserInt32(AMemory Memory, long Address, int Value)
+        {
+            if (Memory.IsMapped(Address))
+            {
+                Memory.WriteInt32ToSharedAddr(Address, Value);
+
+                return true;
+            }
+
+            return false;
+        }
+    }
+}

Ryujinx.HLE/HOS/Kernel/KCoreContext.cs

@@ -0,0 +1,67 @@
+using System;
+
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    class KCoreContext
+    {
+        private KScheduler Scheduler;
+
+        private HleCoreManager CoreManager;
+
+        public bool ContextSwitchNeeded { get; private set; }
+
+        public KThread CurrentThread  { get; private set; }
+        public KThread SelectedThread { get; private set; }
+
+        public KCoreContext(KScheduler Scheduler, HleCoreManager CoreManager)
+        {
+            this.Scheduler   = Scheduler;
+            this.CoreManager = CoreManager;
+        }
+
+        public void SelectThread(KThread Thread)
+        {
+            SelectedThread = Thread;
+
+            if (Thread != null)
+            {
+                Thread.LastScheduledTicks = (uint)Environment.TickCount;
+            }
+
+            ContextSwitchNeeded = true;
+        }
+
+        public void UpdateCurrentThread()
+        {
+            ContextSwitchNeeded = false;
+
+            CurrentThread = SelectedThread;
+        }
+
+        public void ContextSwitch()
+        {
+            ContextSwitchNeeded = false;
+
+            if (CurrentThread != null)
+            {
+                CoreManager.GetThread(CurrentThread.Context.Work).Reset();
+            }
+
+            CurrentThread = SelectedThread;
+
+            if (CurrentThread != null)
+            {
+                CurrentThread.ClearExclusive();
+
+                CoreManager.GetThread(CurrentThread.Context.Work).Set();
+
+                CurrentThread.Context.Execute();
+            }
+        }
+
+        public void RemoveThread(KThread Thread)
+        {
+            //TODO.
+        }
+    }
+}

Ryujinx.HLE/HOS/Kernel/KEvent.cs

@@ -1,4 +1,38 @@
 namespace Ryujinx.HLE.HOS.Kernel
 {
-    class KEvent : KSynchronizationObject { }
+    class KEvent : KSynchronizationObject
+    {
+        private bool Signaled;
+
+        public string Name { get; private set; }
+
+        public KEvent(Horizon System, string Name = "") : base(System)
+        {
+            this.Name = Name;
+        }
+
+        public override void Signal()
+        {
+            System.CriticalSectionLock.Lock();
+
+            if (!Signaled)
+            {
+                Signaled = true;
+
+                base.Signal();
+            }
+
+            System.CriticalSectionLock.Unlock();
+        }
+
+        public void Reset()
+        {
+            Signaled = false;
+        }
+
+        public override bool IsSignaled()
+        {
+            return Signaled;
+        }
+    }
 }

Ryujinx.HLE/HOS/Kernel/KProcessScheduler.cs

@@ -1,370 +0,0 @@
-using Ryujinx.HLE.Logging;
-using System;
-using System.Collections.Concurrent;
-using System.Threading;
-
-namespace Ryujinx.HLE.HOS.Kernel
-{
-    class KProcessScheduler : IDisposable
-    {
-        private ConcurrentDictionary<KThread, SchedulerThread> AllThreads;
-
-        private ThreadQueue WaitingToRun;
-
-        private KThread[] CoreThreads;
-
-        private bool[] CoreReschedule;
-
-        private object SchedLock;
-
-        private Logger Log;
-
-        public KProcessScheduler(Logger Log)
-        {
-            this.Log = Log;
-
-            AllThreads = new ConcurrentDictionary<KThread, SchedulerThread>();
-
-            WaitingToRun = new ThreadQueue();
-
-            CoreThreads = new KThread[4];
-
-            CoreReschedule = new bool[4];
-
-            SchedLock = new object();
-        }
-
-        public void StartThread(KThread Thread)
-        {
-            lock (SchedLock)
-            {
-                SchedulerThread SchedThread = new SchedulerThread(Thread);
-
-                if (!AllThreads.TryAdd(Thread, SchedThread))
-                {
-                    return;
-                }
-
-                if (TryAddToCore(Thread))
-                {
-                    Thread.Thread.Execute();
-
-                    PrintDbgThreadInfo(Thread, "running.");
-                }
-                else
-                {
-                    WaitingToRun.Push(SchedThread);
-
-                    PrintDbgThreadInfo(Thread, "waiting to run.");
-                }
-            }
-        }
-
-        public void RemoveThread(KThread Thread)
-        {
-            PrintDbgThreadInfo(Thread, "exited.");
-
-            lock (SchedLock)
-            {
-                if (AllThreads.TryRemove(Thread, out SchedulerThread SchedThread))
-                {
-                    WaitingToRun.Remove(SchedThread);
-
-                    SchedThread.Dispose();
-                }
-
-                int ActualCore = Thread.ActualCore;
-
-                SchedulerThread NewThread = WaitingToRun.Pop(ActualCore);
-
-                if (NewThread == null)
-                {
-                    Log.PrintDebug(LogClass.KernelScheduler, $"Nothing to run on core {ActualCore}!");
-
-                    CoreThreads[ActualCore] = null;
-
-                    return;
-                }
-
-                NewThread.Thread.ActualCore = ActualCore;
-
-                RunThread(NewThread);
-            }
-        }
-
-        public void SetThreadActivity(KThread Thread, bool Active)
-        {
-            SchedulerThread SchedThread = AllThreads[Thread];
-
-            SchedThread.IsActive = Active;
-
-            if (Active)
-            {
-                SchedThread.WaitActivity.Set();
-            }
-            else
-            {
-                SchedThread.WaitActivity.Reset();
-            }
-        }
-
-        public void EnterWait(KThread Thread, int TimeoutMs = Timeout.Infinite)
-        {
-            SchedulerThread SchedThread = AllThreads[Thread];
-
-            Suspend(Thread);
-
-            SchedThread.WaitSync.WaitOne(TimeoutMs);
-
-            TryResumingExecution(SchedThread);
-        }
-
-        public void WakeUp(KThread Thread)
-        {
-            AllThreads[Thread].WaitSync.Set();
-        }
-
-        public void ForceWakeUp(KThread Thread)
-        {
-            if (AllThreads.TryGetValue(Thread, out SchedulerThread SchedThread))
-            {
-                SchedThread.WaitSync.Set();
-                SchedThread.WaitActivity.Set();
-                SchedThread.WaitSched.Set();
-            }
-        }
-
-        public void ChangeCore(KThread Thread, int IdealCore, int CoreMask)
-        {
-            lock (SchedLock)
-            {
-                if (IdealCore != -3)
-                {
-                    Thread.IdealCore = IdealCore;
-                }
-
-                Thread.CoreMask = CoreMask;
-
-                if (AllThreads.ContainsKey(Thread))
-                {
-                    SetReschedule(Thread.ActualCore);
-
-                    SchedulerThread SchedThread = AllThreads[Thread];
-
-                    //Note: Aways if the thread is on the queue first, and try
-                    //adding to a new core later, to ensure that a thread that
-                    //is already running won't be added to another core.
-                    if (WaitingToRun.HasThread(SchedThread) && TryAddToCore(Thread))
-                    {
-                        WaitingToRun.Remove(SchedThread);
-
-                        RunThread(SchedThread);
-                    }
-                }
-            }
-        }
-
-        public void Suspend(KThread Thread)
-        {
-            lock (SchedLock)
-            {
-                PrintDbgThreadInfo(Thread, "suspended.");
-
-                int ActualCore = Thread.ActualCore;
-
-                CoreReschedule[ActualCore] = false;
-
-                SchedulerThread SchedThread = WaitingToRun.Pop(ActualCore);
-
-                if (SchedThread != null)
-                {
-                    SchedThread.Thread.ActualCore = ActualCore;
-
-                    CoreThreads[ActualCore] = SchedThread.Thread;
-
-                    RunThread(SchedThread);
-                }
-                else
-                {
-                    Log.PrintDebug(LogClass.KernelScheduler, $"Nothing to run on core {Thread.ActualCore}!");
-
-                    CoreThreads[ActualCore] = null;
-                }
-            }
-        }
-
-        public void SetReschedule(int Core)
-        {
-            lock (SchedLock)
-            {
-                CoreReschedule[Core] = true;
-            }
-        }
-
-        public void Reschedule(KThread Thread)
-        {
-            bool NeedsReschedule;
-
-            lock (SchedLock)
-            {
-                int ActualCore = Thread.ActualCore;
-
-                NeedsReschedule = CoreReschedule[ActualCore];
-
-                CoreReschedule[ActualCore] = false;
-            }
-
-            if (NeedsReschedule)
-            {
-                Yield(Thread, Thread.ActualPriority - 1);
-            }
-        }
-
-        public void Yield(KThread Thread)
-        {
-            Yield(Thread, Thread.ActualPriority);
-        }
-
-        private void Yield(KThread Thread, int MinPriority)
-        {
-            PrintDbgThreadInfo(Thread, "yielded execution.");
-
-            lock (SchedLock)
-            {
-                int ActualCore = Thread.ActualCore;
-
-                SchedulerThread NewThread = WaitingToRun.Pop(ActualCore, MinPriority);
-
-                if (NewThread != null)
-                {
-                    NewThread.Thread.ActualCore = ActualCore;
-
-                    CoreThreads[ActualCore] = NewThread.Thread;
-
-                    RunThread(NewThread);
-                }
-                else
-                {
-                    CoreThreads[ActualCore] = null;
-                }
-            }
-
-            Resume(Thread);
-        }
-
-        public void Resume(KThread Thread)
-        {
-            TryResumingExecution(AllThreads[Thread]);
-        }
-
-        private void TryResumingExecution(SchedulerThread SchedThread)
-        {
-            KThread Thread = SchedThread.Thread;
-
-            PrintDbgThreadInfo(Thread, "trying to resume...");
-
-            SchedThread.WaitActivity.WaitOne();
-
-            lock (SchedLock)
-            {
-                if (TryAddToCore(Thread))
-                {
-                    PrintDbgThreadInfo(Thread, "resuming execution...");
-
-                    return;
-                }
-
-                WaitingToRun.Push(SchedThread);
-
-                SetReschedule(Thread.ProcessorId);
-
-                PrintDbgThreadInfo(Thread, "entering wait state...");
-            }
-
-            SchedThread.WaitSched.WaitOne();
-
-            PrintDbgThreadInfo(Thread, "resuming execution...");
-        }
-
-        private void RunThread(SchedulerThread SchedThread)
-        {
-            if (!SchedThread.Thread.Thread.Execute())
-            {
-                PrintDbgThreadInfo(SchedThread.Thread, "waked.");
-
-                SchedThread.WaitSched.Set();
-            }
-            else
-            {
-                PrintDbgThreadInfo(SchedThread.Thread, "running.");
-            }
-        }
-
-        public void Resort(KThread Thread)
-        {
-            if (AllThreads.TryGetValue(Thread, out SchedulerThread SchedThread))
-            {
-                WaitingToRun.Resort(SchedThread);
-            }
-        }
-
-        private bool TryAddToCore(KThread Thread)
-        {
-            //First, try running it on Ideal Core.
-            int IdealCore = Thread.IdealCore;
-
-            if (IdealCore != -1 && CoreThreads[IdealCore] == null)
-            {
-                Thread.ActualCore = IdealCore;
-
-                CoreThreads[IdealCore] = Thread;
-
-                return true;
-            }
-
-            //If that fails, then try running on any core allowed by Core Mask.
-            int CoreMask = Thread.CoreMask;
-
-            for (int Core = 0; Core < CoreThreads.Length; Core++, CoreMask >>= 1)
-            {
-                if ((CoreMask & 1) != 0 && CoreThreads[Core] == null)
-                {
-                    Thread.ActualCore = Core;
-
-                    CoreThreads[Core] = Thread;
-
-                    return true;
-                }
-            }
-
-            return false;
-        }
-
-        private void PrintDbgThreadInfo(KThread Thread, string Message)
-        {
-            Log.PrintDebug(LogClass.KernelScheduler, "(" +
-                "ThreadId = "       + Thread.ThreadId                + ", " +
-                "CoreMask = 0x"     + Thread.CoreMask.ToString("x1") + ", " +
-                "ActualCore = "     + Thread.ActualCore              + ", " +
-                "IdealCore = "      + Thread.IdealCore               + ", " +
-                "ActualPriority = " + Thread.ActualPriority          + ", " +
-                "WantedPriority = " + Thread.WantedPriority          + ") " + Message);
-        }
-
-        public void Dispose()
-        {
-            Dispose(true);
-        }
-
-        protected virtual void Dispose(bool Disposing)
-        {
-            if (Disposing)
-            {
-                foreach (SchedulerThread SchedThread in AllThreads.Values)
-                {
-                    SchedThread.Dispose();
-                }
-            }
-        }
-    }
-}

Ryujinx.HLE/HOS/Kernel/KRecursiveLock.cs

@@ -0,0 +1,93 @@
+using ChocolArm64;
+using System.Threading;
+
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    class KRecursiveLock
+    {
+        private Horizon System;
+
+        public object LockObj { get; private set; }
+
+        private int RecursionCount;
+
+        public KRecursiveLock(Horizon System)
+        {
+            this.System = System;
+
+            LockObj = new object();
+        }
+
+        public void Lock()
+        {
+            Monitor.Enter(LockObj);
+
+            RecursionCount++;
+        }
+
+        public void Unlock()
+        {
+            if (RecursionCount == 0)
+            {
+                return;
+            }
+
+            bool DoContextSwitch = false;
+
+            if (--RecursionCount == 0)
+            {
+                if (System.Scheduler.ThreadReselectionRequested)
+                {
+                    System.Scheduler.SelectThreads();
+                }
+
+                Monitor.Exit(LockObj);
+
+                if (System.Scheduler.MultiCoreScheduling)
+                {
+                    lock (System.Scheduler.CoreContexts)
+                    {
+                        for (int Core = 0; Core < KScheduler.CpuCoresCount; Core++)
+                        {
+                            KCoreContext CoreContext = System.Scheduler.CoreContexts[Core];
+
+                            if (CoreContext.ContextSwitchNeeded)
+                            {
+                                AThread CurrentHleThread = CoreContext.CurrentThread?.Context;
+
+                                if (CurrentHleThread == null)
+                                {
+                                    //Nothing is running, we can perform the context switch immediately.
+                                    CoreContext.ContextSwitch();
+                                }
+                                else if (CurrentHleThread.IsCurrentThread())
+                                {
+                                    //Thread running on the current core, context switch will block.
+                                    DoContextSwitch = true;
+                                }
+                                else
+                                {
+                                    //Thread running on another core, request a interrupt.
+                                    CurrentHleThread.RequestInterrupt();
+                                }
+                            }
+                        }
+                    }
+                }
+                else
+                {
+                    DoContextSwitch = true;
+                }
+            }
+            else
+            {
+                Monitor.Exit(LockObj);
+            }
+
+            if (DoContextSwitch)
+            {
+                System.Scheduler.ContextSwitch();
+            }
+        }
+    }
+}

Ryujinx.HLE/HOS/Kernel/KScheduler.cs

@@ -0,0 +1,235 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    partial class KScheduler : IDisposable
+    {
+        public const int PrioritiesCount = 64;
+        public const int CpuCoresCount   = 4;
+
+        private const int PreemptionPriorityCores012 = 59;
+        private const int PreemptionPriorityCore3    = 63;
+
+        private Horizon System;
+
+        public KSchedulingData SchedulingData { get; private set; }
+
+        public KCoreContext[] CoreContexts { get; private set; }
+
+        public bool ThreadReselectionRequested { get; set; }
+
+        public KScheduler(Horizon System)
+        {
+            this.System = System;
+
+            SchedulingData = new KSchedulingData();
+
+            CoreManager = new HleCoreManager();
+
+            CoreContexts = new KCoreContext[CpuCoresCount];
+
+            for (int Core = 0; Core < CpuCoresCount; Core++)
+            {
+                CoreContexts[Core] = new KCoreContext(this, CoreManager);
+            }
+
+            Thread PreemptionThread = new Thread(PreemptCurrentThread);
+
+            KeepPreempting = true;
+
+            PreemptionThread.Start();
+        }
+
+        private void PreemptThreads()
+        {
+            System.CriticalSectionLock.Lock();
+
+            PreemptThread(PreemptionPriorityCores012, 0);
+            PreemptThread(PreemptionPriorityCores012, 1);
+            PreemptThread(PreemptionPriorityCores012, 2);
+            PreemptThread(PreemptionPriorityCore3,    3);
+
+            System.CriticalSectionLock.Unlock();
+        }
+
+        private void PreemptThread(int Prio, int Core)
+        {
+            IEnumerable<KThread> ScheduledThreads = SchedulingData.ScheduledThreads(Core);
+
+            KThread SelectedThread = ScheduledThreads.FirstOrDefault(x => x.DynamicPriority == Prio);
+
+            //Yield priority queue.
+            if (SelectedThread != null)
+            {
+                SchedulingData.Reschedule(Prio, Core, SelectedThread);
+            }
+
+            IEnumerable<KThread> SuitableCandidates()
+            {
+                foreach (KThread Thread in SchedulingData.SuggestedThreads(Core))
+                {
+                    int SrcCore = Thread.CurrentCore;
+
+                    if (SrcCore >= 0)
+                    {
+                        KThread HighestPrioSrcCore = SchedulingData.ScheduledThreads(SrcCore).FirstOrDefault();
+
+                        if (HighestPrioSrcCore != null && HighestPrioSrcCore.DynamicPriority < 2)
+                        {
+                            break;
+                        }
+
+                        if (HighestPrioSrcCore == Thread)
+                        {
+                            continue;
+                        }
+                    }
+
+                    //If the candidate was scheduled after the current thread, then it's not worth it.
+                    if (SelectedThread == null || SelectedThread.LastScheduledTicks >= Thread.LastScheduledTicks)
+                    {
+                        yield return Thread;
+                    }
+                }
+            }
+
+            //Select candidate threads that could run on this core.
+            //Only take into account threads that are not yet selected.
+            KThread Dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority == Prio);
+
+            if (Dst != null)
+            {
+                SchedulingData.TransferToCore(Prio, Core, Dst);
+
+                SelectedThread = Dst;
+            }
+
+            //If the priority of the currently selected thread is lower than preemption priority,
+            //then allow threads with lower priorities to be selected aswell.
+            if (SelectedThread != null && SelectedThread.DynamicPriority > Prio)
+            {
+                Func<KThread, bool> Predicate = x => x.DynamicPriority >= SelectedThread.DynamicPriority;
+
+                Dst = SuitableCandidates().FirstOrDefault(Predicate);
+
+                if (Dst != null)
+                {
+                    SchedulingData.TransferToCore(Dst.DynamicPriority, Core, Dst);
+                }
+            }
+
+            ThreadReselectionRequested = true;
+        }
+
+        public void SelectThreads()
+        {
+            ThreadReselectionRequested = false;
+
+            for (int Core = 0; Core < CpuCoresCount; Core++)
+            {
+                KThread Thread = SchedulingData.ScheduledThreads(Core).FirstOrDefault();
+
+                CoreContexts[Core].SelectThread(Thread);
+            }
+
+            for (int Core = 0; Core < CpuCoresCount; Core++)
+            {
+                //If the core is not idle (there's already a thread running on it),
+                //then we don't need to attempt load balancing.
+                if (SchedulingData.ScheduledThreads(Core).Any())
+                {
+                    continue;
+                }
+
+                int[] SrcCoresHighestPrioThreads = new int[CpuCoresCount];
+
+                int SrcCoresHighestPrioThreadsCount = 0;
+
+                KThread Dst = null;
+
+                //Select candidate threads that could run on this core.
+                //Give preference to threads that are not yet selected.
+                foreach (KThread Thread in SchedulingData.SuggestedThreads(Core))
+                {
+                    if (Thread.CurrentCore < 0 || Thread != CoreContexts[Thread.CurrentCore].SelectedThread)
+                    {
+                        Dst = Thread;
+
+                        break;
+                    }
+
+                    SrcCoresHighestPrioThreads[SrcCoresHighestPrioThreadsCount++] = Thread.CurrentCore;
+                }
+
+                //Not yet selected candidate found.
+                if (Dst != null)
+                {
+                    //Priorities < 2 are used for the kernel message dispatching
+                    //threads, we should skip load balancing entirely.
+                    if (Dst.DynamicPriority >= 2)
+                    {
+                        SchedulingData.TransferToCore(Dst.DynamicPriority, Core, Dst);
+
+                        CoreContexts[Core].SelectThread(Dst);
+                    }
+
+                    continue;
+                }
+
+                //All candiates are already selected, choose the best one
+                //(the first one that doesn't make the source core idle if moved).
+                for (int Index = 0; Index < SrcCoresHighestPrioThreadsCount; Index++)
+                {
+                    int SrcCore = SrcCoresHighestPrioThreads[Index];
+
+                    KThread Src = SchedulingData.ScheduledThreads(SrcCore).ElementAtOrDefault(1);
+
+                    if (Src != null)
+                    {
+                        //Run the second thread on the queue on the source core,
+                        //move the first one to the current core.
+                        KThread OrigSelectedCoreSrc = CoreContexts[SrcCore].SelectedThread;
+
+                        CoreContexts[SrcCore].SelectThread(Src);
+
+                        SchedulingData.TransferToCore(OrigSelectedCoreSrc.DynamicPriority, Core, OrigSelectedCoreSrc);
+
+                        CoreContexts[Core].SelectThread(OrigSelectedCoreSrc);
+                    }
+                }
+            }
+        }
+
+        public KThread GetCurrentThread()
+        {
+            lock (CoreContexts)
+            {
+                for (int Core = 0; Core < CpuCoresCount; Core++)
+                {
+                    if (CoreContexts[Core].CurrentThread?.Context.IsCurrentThread() ?? false)
+                    {
+                        return CoreContexts[Core].CurrentThread;
+                    }
+                }
+            }
+
+            throw new InvalidOperationException("Current thread is not scheduled!");
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+        }
+
+        protected virtual void Dispose(bool Disposing)
+        {
+            if (Disposing)
+            {
+                KeepPreempting = false;
+            }
+        }
+    }
+}

Ryujinx.HLE/HOS/Kernel/KSchedulingData.cs

@@ -0,0 +1,207 @@
+using System.Collections.Generic;
+
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    class KSchedulingData
+    {
+        private LinkedList<KThread>[][] ScheduledThreadsPerPrioPerCore;
+        private LinkedList<KThread>[][] SuggestedThreadsPerPrioPerCore;
+
+        private long[] ScheduledPrioritiesPerCore;
+        private long[] SuggestedPrioritiesPerCore;
+
+        public KSchedulingData()
+        {
+            SuggestedThreadsPerPrioPerCore = new LinkedList<KThread>[KScheduler.PrioritiesCount][];
+            ScheduledThreadsPerPrioPerCore = new LinkedList<KThread>[KScheduler.PrioritiesCount][];
+
+            for (int Prio = 0; Prio < KScheduler.PrioritiesCount; Prio++)
+            {
+                SuggestedThreadsPerPrioPerCore[Prio] = new LinkedList<KThread>[KScheduler.CpuCoresCount];
+                ScheduledThreadsPerPrioPerCore[Prio] = new LinkedList<KThread>[KScheduler.CpuCoresCount];
+
+                for (int Core = 0; Core < KScheduler.CpuCoresCount; Core++)
+                {
+                    SuggestedThreadsPerPrioPerCore[Prio][Core] = new LinkedList<KThread>();
+                    ScheduledThreadsPerPrioPerCore[Prio][Core] = new LinkedList<KThread>();
+                }
+            }
+
+            ScheduledPrioritiesPerCore = new long[KScheduler.CpuCoresCount];
+            SuggestedPrioritiesPerCore = new long[KScheduler.CpuCoresCount];
+        }
+
+        public IEnumerable<KThread> SuggestedThreads(int Core)
+        {
+            return Iterate(SuggestedThreadsPerPrioPerCore, SuggestedPrioritiesPerCore, Core);
+        }
+
+        public IEnumerable<KThread> ScheduledThreads(int Core)
+        {
+            return Iterate(ScheduledThreadsPerPrioPerCore, ScheduledPrioritiesPerCore, Core);
+        }
+
+        private IEnumerable<KThread> Iterate(LinkedList<KThread>[][] ListPerPrioPerCore, long[] Prios, int Core)
+        {
+            long PrioMask = Prios[Core];
+
+            int Prio = CountTrailingZeros(PrioMask);
+
+            PrioMask &= ~(1L << Prio);
+
+            while (Prio < KScheduler.PrioritiesCount)
+            {
+                LinkedList<KThread> List = ListPerPrioPerCore[Prio][Core];
+
+                LinkedListNode<KThread> Node = List.First;
+
+                while (Node != null)
+                {
+                    yield return Node.Value;
+
+                    Node = Node.Next;
+                }
+
+                Prio = CountTrailingZeros(PrioMask);
+
+                PrioMask &= ~(1L << Prio);
+            }
+        }
+
+        private int CountTrailingZeros(long Value)
+        {
+            int Count = 0;
+
+            while (((Value >> Count) & 0xf) == 0 && Count < 64)
+            {
+                Count += 4;
+            }
+
+            while (((Value >> Count) & 1) == 0 && Count < 64)
+            {
+                Count++;
+            }
+
+            return Count;
+        }
+
+        public void TransferToCore(int Prio, int DstCore, KThread Thread)
+        {
+            bool Schedulable = Thread.DynamicPriority < KScheduler.PrioritiesCount;
+
+            int SrcCore = Thread.CurrentCore;
+
+            Thread.CurrentCore = DstCore;
+
+            if (SrcCore == DstCore || !Schedulable)
+            {
+                return;
+            }
+
+            if (SrcCore >= 0)
+            {
+                Unschedule(Prio, SrcCore, Thread);
+            }
+
+            if (DstCore >= 0)
+            {
+                Unsuggest(Prio, DstCore, Thread);
+                Schedule(Prio, DstCore, Thread);
+            }
+
+            if (SrcCore >= 0)
+            {
+                Suggest(Prio, SrcCore, Thread);
+            }
+        }
+
+        public void Suggest(int Prio, int Core, KThread Thread)
+        {
+            if (Prio >= KScheduler.PrioritiesCount)
+            {
+                return;
+            }
+
+            Thread.SiblingsPerCore[Core] = SuggestedQueue(Prio, Core).AddFirst(Thread);
+
+            SuggestedPrioritiesPerCore[Core] |= 1L << Prio;
+        }
+
+        public void Unsuggest(int Prio, int Core, KThread Thread)
+        {
+            if (Prio >= KScheduler.PrioritiesCount)
+            {
+                return;
+            }
+
+            LinkedList<KThread> Queue = SuggestedQueue(Prio, Core);
+
+            Queue.Remove(Thread.SiblingsPerCore[Core]);
+
+            if (Queue.First == null)
+            {
+                SuggestedPrioritiesPerCore[Core] &= ~(1L << Prio);
+            }
+        }
+
+        public void Schedule(int Prio, int Core, KThread Thread)
+        {
+            if (Prio >= KScheduler.PrioritiesCount)
+            {
+                return;
+            }
+
+            Thread.SiblingsPerCore[Core] = ScheduledQueue(Prio, Core).AddLast(Thread);
+
+            ScheduledPrioritiesPerCore[Core] |= 1L << Prio;
+        }
+
+        public void SchedulePrepend(int Prio, int Core, KThread Thread)
+        {
+            if (Prio >= KScheduler.PrioritiesCount)
+            {
+                return;
+            }
+
+            Thread.SiblingsPerCore[Core] = ScheduledQueue(Prio, Core).AddFirst(Thread);
+
+            ScheduledPrioritiesPerCore[Core] |= 1L << Prio;
+        }
+
+        public void Reschedule(int Prio, int Core, KThread Thread)
+        {
+            LinkedList<KThread> Queue = ScheduledQueue(Prio, Core);
+
+            Queue.Remove(Thread.SiblingsPerCore[Core]);
+
+            Thread.SiblingsPerCore[Core] = Queue.AddLast(Thread);
+        }
+
+        public void Unschedule(int Prio, int Core, KThread Thread)
+        {
+            if (Prio >= KScheduler.PrioritiesCount)
+            {
+                return;
+            }
+
+            LinkedList<KThread> Queue = ScheduledQueue(Prio, Core);
+
+            Queue.Remove(Thread.SiblingsPerCore[Core]);
+
+            if (Queue.First == null)
+            {
+                ScheduledPrioritiesPerCore[Core] &= ~(1L << Prio);
+            }
+        }
+
+        private LinkedList<KThread> SuggestedQueue(int Prio, int Core)
+        {
+            return SuggestedThreadsPerPrioPerCore[Prio][Core];
+        }
+
+        private LinkedList<KThread> ScheduledQueue(int Prio, int Core)
+        {
+            return ScheduledThreadsPerPrioPerCore[Prio][Core];
+        }
+    }
+}

Ryujinx.HLE/HOS/Kernel/KSynchronization.cs

@@ -0,0 +1,135 @@
+using System.Collections.Generic;
+
+using static Ryujinx.HLE.HOS.ErrorCode;
+
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    class KSynchronization
+    {
+        private Horizon System;
+
+        public KSynchronization(Horizon System)
+        {
+            this.System = System;
+        }
+
+        public long WaitFor(KSynchronizationObject[] SyncObjs, long Timeout, ref int HndIndex)
+        {
+            long Result = MakeError(ErrorModule.Kernel, KernelErr.Timeout);
+
+            System.CriticalSectionLock.Lock();
+
+            //Check if objects are already signaled before waiting.
+            for (int Index = 0; Index < SyncObjs.Length; Index++)
+            {
+                if (!SyncObjs[Index].IsSignaled())
+                {
+                    continue;
+                }
+
+                HndIndex = Index;
+
+                System.CriticalSectionLock.Unlock();
+
+                return 0;
+            }
+
+            if (Timeout == 0)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return Result;
+            }
+
+            KThread CurrentThread = System.Scheduler.GetCurrentThread();
+
+            if (CurrentThread.ShallBeTerminated ||
+                CurrentThread.SchedFlags == ThreadSchedState.TerminationPending)
+            {
+                Result = MakeError(ErrorModule.Kernel, KernelErr.ThreadTerminating);
+            }
+            else if (CurrentThread.SyncCancelled)
+            {
+                CurrentThread.SyncCancelled = false;
+
+                Result = MakeError(ErrorModule.Kernel, KernelErr.Cancelled);
+            }
+            else
+            {
+                LinkedListNode<KThread>[] SyncNodes = new LinkedListNode<KThread>[SyncObjs.Length];
+
+                for (int Index = 0; Index < SyncObjs.Length; Index++)
+                {
+                    SyncNodes[Index] = SyncObjs[Index].AddWaitingThread(CurrentThread);
+                }
+
+                CurrentThread.WaitingSync   = true;
+                CurrentThread.SignaledObj   = null;
+                CurrentThread.ObjSyncResult = (int)Result;
+
+                CurrentThread.Reschedule(ThreadSchedState.Paused);
+
+                if (Timeout > 0)
+                {
+                    System.TimeManager.ScheduleFutureInvocation(CurrentThread, Timeout);
+                }
+
+                System.CriticalSectionLock.Unlock();
+
+                CurrentThread.WaitingSync = false;
+
+                if (Timeout > 0)
+                {
+                    System.TimeManager.UnscheduleFutureInvocation(CurrentThread);
+                }
+
+                System.CriticalSectionLock.Lock();
+
+                Result = (uint)CurrentThread.ObjSyncResult;
+
+                HndIndex = -1;
+
+                for (int Index = 0; Index < SyncObjs.Length; Index++)
+                {
+                    SyncObjs[Index].RemoveWaitingThread(SyncNodes[Index]);
+
+                    if (SyncObjs[Index] == CurrentThread.SignaledObj)
+                    {
+                        HndIndex = Index;
+                    }
+                }
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            return Result;
+        }
+
+        public void SignalObject(KSynchronizationObject SyncObj)
+        {
+            System.CriticalSectionLock.Lock();
+
+            if (SyncObj.IsSignaled())
+            {
+                LinkedListNode<KThread> Node = SyncObj.WaitingThreads.First;
+
+                while (Node != null)
+                {
+                    KThread Thread = Node.Value;
+
+                    if ((Thread.SchedFlags & ThreadSchedState.LowNibbleMask) == ThreadSchedState.Paused)
+                    {
+                        Thread.SignaledObj   = SyncObj;
+                        Thread.ObjSyncResult = 0;
+
+                        Thread.Reschedule(ThreadSchedState.Running);
+                    }
+
+                    Node = Node.Next;
+                }
+            }
+
+            System.CriticalSectionLock.Unlock();
+        }
+    }
+}

Ryujinx.HLE/HOS/Kernel/KSynchronizationObject.cs

@@ -1,28 +1,38 @@
-using System;
-using System.Threading;
+using System.Collections.Generic;
 
 namespace Ryujinx.HLE.HOS.Kernel
 {
-    class KSynchronizationObject : IDisposable
+    class KSynchronizationObject
     {
-        public ManualResetEvent WaitEvent { get; private set; }
+        public LinkedList<KThread> WaitingThreads;
 
-        public KSynchronizationObject()
+        protected Horizon System;
+
+        public KSynchronizationObject(Horizon System)
         {
-            WaitEvent = new ManualResetEvent(false);
+            this.System = System;
+
+            WaitingThreads = new LinkedList<KThread>();
         }
 
-        public void Dispose()
+        public LinkedListNode<KThread> AddWaitingThread(KThread Thread)
         {
-            Dispose(true);
+            return WaitingThreads.AddLast(Thread);
         }
 
-        protected virtual void Dispose(bool Disposing)
+        public void RemoveWaitingThread(LinkedListNode<KThread> Node)
         {
-            if (Disposing)
+            WaitingThreads.Remove(Node);
+        }
+
+        public virtual void Signal()
         {
-                WaitEvent.Dispose();
-            }
+            System.Synchronization.SignalObject(this);
+        }
+
+        public virtual bool IsSignaled()
+        {
+            return false;
         }
     }
 }

Ryujinx.HLE/HOS/Kernel/KThread.cs

@@ -1,98 +1,883 @@
 using ChocolArm64;
+using System;
 using System.Collections.Generic;
+using System.Linq;
+
+using static Ryujinx.HLE.HOS.ErrorCode;
 
 namespace Ryujinx.HLE.HOS.Kernel
 {
-    class KThread : KSynchronizationObject
+    class KThread : KSynchronizationObject, IKFutureSchedulerObject
     {
-        public AThread Thread { get; private set; }
+        public AThread Context { get; private set; }
 
-        public int CoreMask { get; set; }
-
-        public long MutexAddress       { get; set; }
-        public long CondVarAddress     { get; set; }
-        public long ArbiterWaitAddress { get; set; }
-
-        public bool CondVarSignaled { get; set; }
-        public bool ArbiterSignaled { get; set; }
-
-        private Process Process;
-
-        public List<KThread> MutexWaiters { get; private set; }
-
-        public KThread MutexOwner { get; set; }
-
-        public int ActualPriority { get; private set; }
-        public int WantedPriority { get; private set; }
-
-        public int ActualCore  { get; set; }
-        public int ProcessorId { get; set; }
-        public int IdealCore   { get; set; }
-
-        public int WaitHandle { get; set; }
-
-        public long LastPc { get; set; }
+        public long AffinityMask { get; set; }
 
         public int ThreadId { get; private set; }
 
+        public KSynchronizationObject SignaledObj;
+
+        public long CondVarAddress { get; set; }
+        public long MutexAddress   { get; set; }
+
+        public Process Owner { get; private set; }
+
+        public long LastScheduledTicks { get; set; }
+
+        public LinkedListNode<KThread>[] SiblingsPerCore { get; private set; }
+
+        private LinkedListNode<KThread> WithholderNode;
+
+        private LinkedList<KThread>     MutexWaiters;
+        private LinkedListNode<KThread> MutexWaiterNode;
+
+        public KThread MutexOwner { get; private set; }
+
+        public int ThreadHandleForUserMutex { get; set; }
+
+        private ThreadSchedState ForcePauseFlags;
+
+        public int ObjSyncResult { get; set; }
+
+        public int DynamicPriority { get; set; }
+        public int CurrentCore     { get; set; }
+        public int BasePriority    { get; set; }
+        public int PreferredCore   { get; set; }
+
+        private long AffinityMaskOverride;
+        private int  PreferredCoreOverride;
+        private int  AffinityOverrideCount;
+
+        public ThreadSchedState SchedFlags { get; private set; }
+
+        public bool ShallBeTerminated { get; private set; }
+
+        public bool SyncCancelled { get; set; }
+        public bool WaitingSync   { get; set; }
+
+        private bool HasExited;
+
+        public bool WaitingInArbitration { get; set; }
+
+        private KScheduler Scheduler;
+
+        private KSchedulingData SchedulingData;
+
+        public long LastPc { get; set; }
+
         public KThread(
             AThread Thread,
             Process Process,
+            Horizon System,
             int     ProcessorId,
             int     Priority,
-            int     ThreadId)
+            int     ThreadId) : base(System)
         {
-            this.Thread      = Thread;
-            this.Process     = Process;
-            this.ProcessorId = ProcessorId;
-            this.IdealCore   = ProcessorId;
             this.ThreadId = ThreadId;
 
-            MutexWaiters = new List<KThread>();
+            Context        = Thread;
+            Owner          = Process;
+            PreferredCore  = ProcessorId;
+            Scheduler      = System.Scheduler;
+            SchedulingData = System.Scheduler.SchedulingData;
 
-            CoreMask = 1 << ProcessorId;
+            SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
 
-            ActualPriority = WantedPriority = Priority;
+            MutexWaiters = new LinkedList<KThread>();
+
+            AffinityMask = 1 << ProcessorId;
+
+            DynamicPriority = BasePriority = Priority;
+
+            CurrentCore = PreferredCore;
+        }
+
+        public long Start()
+        {
+            long Result = MakeError(ErrorModule.Kernel, KernelErr.ThreadTerminating);
+
+            System.CriticalSectionLock.Lock();
+
+            if (!ShallBeTerminated)
+            {
+                KThread CurrentThread = System.Scheduler.GetCurrentThread();
+
+                while (SchedFlags               != ThreadSchedState.TerminationPending &&
+                       CurrentThread.SchedFlags != ThreadSchedState.TerminationPending &&
+                       !CurrentThread.ShallBeTerminated)
+                {
+                    if ((SchedFlags & ThreadSchedState.LowNibbleMask) != ThreadSchedState.None)
+                    {
+                        Result = MakeError(ErrorModule.Kernel, KernelErr.InvalidState);
+
+                        break;
+                    }
+
+                    if (CurrentThread.ForcePauseFlags == ThreadSchedState.None)
+                    {
+                        if (Owner != null && ForcePauseFlags != ThreadSchedState.None)
+                        {
+                            CombineForcePauseFlags();
+                        }
+
+                        SetNewSchedFlags(ThreadSchedState.Running);
+
+                        Result = 0;
+
+                        break;
+                    }
+                    else
+                    {
+                        CurrentThread.CombineForcePauseFlags();
+
+                        System.CriticalSectionLock.Unlock();
+                        System.CriticalSectionLock.Lock();
+
+                        if (CurrentThread.ShallBeTerminated)
+                        {
+                            break;
+                        }
+                    }
+                }
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            return Result;
+        }
+
+        public void Exit()
+        {
+            System.CriticalSectionLock.Lock();
+
+            ForcePauseFlags &= ~ThreadSchedState.ExceptionalMask;
+
+            ExitImpl();
+
+            System.CriticalSectionLock.Unlock();
+        }
+
+        private void ExitImpl()
+        {
+            System.CriticalSectionLock.Lock();
+
+            SetNewSchedFlags(ThreadSchedState.TerminationPending);
+
+            HasExited = true;
+
+            Signal();
+
+            System.CriticalSectionLock.Unlock();
+        }
+
+        public long Sleep(long Timeout)
+        {
+            System.CriticalSectionLock.Lock();
+
+            if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                return MakeError(ErrorModule.Kernel, KernelErr.ThreadTerminating);
+            }
+
+            SetNewSchedFlags(ThreadSchedState.Paused);
+
+            if (Timeout > 0)
+            {
+                System.TimeManager.ScheduleFutureInvocation(this, Timeout);
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            if (Timeout > 0)
+            {
+                System.TimeManager.UnscheduleFutureInvocation(this);
+            }
+
+            return 0;
+        }
+
+        public void Yield()
+        {
+            System.CriticalSectionLock.Lock();
+
+            if (SchedFlags != ThreadSchedState.Running)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                System.Scheduler.ContextSwitch();
+
+                return;
+            }
+
+            if (DynamicPriority < KScheduler.PrioritiesCount)
+            {
+                //Move current thread to the end of the queue.
+                SchedulingData.Reschedule(DynamicPriority, CurrentCore, this);
+            }
+
+            Scheduler.ThreadReselectionRequested = true;
+
+            System.CriticalSectionLock.Unlock();
+
+            System.Scheduler.ContextSwitch();
+        }
+
+        public void YieldWithLoadBalancing()
+        {
+            int Prio = DynamicPriority;
+            int Core = CurrentCore;
+
+            System.CriticalSectionLock.Lock();
+
+            if (SchedFlags != ThreadSchedState.Running)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                System.Scheduler.ContextSwitch();
+
+                return;
+            }
+
+            KThread NextThreadOnCurrentQueue = null;
+
+            if (DynamicPriority < KScheduler.PrioritiesCount)
+            {
+                //Move current thread to the end of the queue.
+                SchedulingData.Reschedule(Prio, Core, this);
+
+                Func<KThread, bool> Predicate = x => x.DynamicPriority == Prio;
+
+                NextThreadOnCurrentQueue = SchedulingData.ScheduledThreads(Core).FirstOrDefault(Predicate);
+            }
+
+            IEnumerable<KThread> SuitableCandidates()
+            {
+                foreach (KThread Thread in SchedulingData.SuggestedThreads(Core))
+                {
+                    int SrcCore = Thread.CurrentCore;
+
+                    if (SrcCore >= 0)
+                    {
+                        KThread SelectedSrcCore = Scheduler.CoreContexts[SrcCore].SelectedThread;
+
+                        if (SelectedSrcCore == Thread || ((SelectedSrcCore?.DynamicPriority ?? 2) < 2))
+                        {
+                            continue;
+                        }
+                    }
+
+                    //If the candidate was scheduled after the current thread, then it's not worth it,
+                    //unless the priority is higher than the current one.
+                    if (NextThreadOnCurrentQueue.LastScheduledTicks >= Thread.LastScheduledTicks ||
+                        NextThreadOnCurrentQueue.DynamicPriority    <  Thread.DynamicPriority)
+                    {
+                        yield return Thread;
+                    }
+                }
+            }
+
+            KThread Dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority <= Prio);
+
+            if (Dst != null)
+            {
+                SchedulingData.TransferToCore(Dst.DynamicPriority, Core, Dst);
+
+                Scheduler.ThreadReselectionRequested = true;
+            }
+
+            if (this != NextThreadOnCurrentQueue)
+            {
+                Scheduler.ThreadReselectionRequested = true;
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            System.Scheduler.ContextSwitch();
+        }
+
+        public void YieldAndWaitForLoadBalancing()
+        {
+            System.CriticalSectionLock.Lock();
+
+            if (SchedFlags != ThreadSchedState.Running)
+            {
+                System.CriticalSectionLock.Unlock();
+
+                System.Scheduler.ContextSwitch();
+
+                return;
+            }
+
+            int Core = CurrentCore;
+
+            SchedulingData.TransferToCore(DynamicPriority, -1, this);
+
+            KThread SelectedThread = null;
+
+            if (!SchedulingData.ScheduledThreads(Core).Any())
+            {
+                foreach (KThread Thread in SchedulingData.SuggestedThreads(Core))
+                {
+                    if (Thread.CurrentCore < 0)
+                    {
+                        continue;
+                    }
+
+                    KThread FirstCandidate = SchedulingData.ScheduledThreads(Thread.CurrentCore).FirstOrDefault();
+
+                    if (FirstCandidate == Thread)
+                    {
+                        continue;
+                    }
+
+                    if (FirstCandidate == null || FirstCandidate.DynamicPriority >= 2)
+                    {
+                        SchedulingData.TransferToCore(Thread.DynamicPriority, Core, Thread);
+
+                        SelectedThread = Thread;
+                    }
+
+                    break;
+                }
+            }
+
+            if (SelectedThread != this)
+            {
+                Scheduler.ThreadReselectionRequested = true;
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            System.Scheduler.ContextSwitch();
         }
 
         public void SetPriority(int Priority)
         {
-            WantedPriority = Priority;
+            System.CriticalSectionLock.Lock();
 
-            UpdatePriority();
+            BasePriority = Priority;
+
+            UpdatePriorityInheritance();
+
+            System.CriticalSectionLock.Unlock();
         }
 
-        public void UpdatePriority()
+        public long SetActivity(bool Pause)
         {
-            bool PriorityChanged;
+            long Result = 0;
 
-            lock (Process.ThreadSyncLock)
+            System.CriticalSectionLock.Lock();
+
+            ThreadSchedState LowNibble = SchedFlags & ThreadSchedState.LowNibbleMask;
+
+            if (LowNibble != ThreadSchedState.Paused && LowNibble != ThreadSchedState.Running)
             {
-                int OldPriority = ActualPriority;
+                System.CriticalSectionLock.Unlock();
 
-                int CurrPriority = WantedPriority;
+                return MakeError(ErrorModule.Kernel, KernelErr.InvalidState);
+            }
 
-                foreach (KThread Thread in MutexWaiters)
+            System.CriticalSectionLock.Lock();
+
+            if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
             {
-                    int WantedPriority = Thread.WantedPriority;
-
-                    if (CurrPriority > WantedPriority)
+                if (Pause)
                 {
-                        CurrPriority = WantedPriority;
-                    }
-                }
-
-                PriorityChanged = CurrPriority != OldPriority;
-
-                ActualPriority = CurrPriority;
-            }
-
-            if (PriorityChanged)
+                    //Pause, the force pause flag should be clear (thread is NOT paused).
+                    if ((ForcePauseFlags & ThreadSchedState.ForcePauseFlag) == 0)
                     {
-                Process.Scheduler.Resort(this);
+                        ForcePauseFlags |= ThreadSchedState.ForcePauseFlag;
 
-                MutexOwner?.UpdatePriority();
+                        CombineForcePauseFlags();
+                    }
+                    else
+                    {
+                        Result = MakeError(ErrorModule.Kernel, KernelErr.InvalidState);
+                    }
+                }
+                else
+                {
+                    //Unpause, the force pause flag should be set (thread is paused).
+                    if ((ForcePauseFlags & ThreadSchedState.ForcePauseFlag) != 0)
+                    {
+                        ThreadSchedState OldForcePauseFlags = ForcePauseFlags;
+
+                        ForcePauseFlags &= ~ThreadSchedState.ForcePauseFlag;
+
+                        if ((OldForcePauseFlags & ~ThreadSchedState.ForcePauseFlag) == ThreadSchedState.None)
+                        {
+                            ThreadSchedState OldSchedFlags = SchedFlags;
+
+                            SchedFlags &= ThreadSchedState.LowNibbleMask;
+
+                            AdjustScheduling(OldSchedFlags);
+                        }
+                    }
+                    else
+                    {
+                        Result = MakeError(ErrorModule.Kernel, KernelErr.InvalidState);
                     }
                 }
             }
+
+            System.CriticalSectionLock.Unlock();
+            System.CriticalSectionLock.Unlock();
+
+            return Result;
+        }
+
+        public void CancelSynchronization()
+        {
+            System.CriticalSectionLock.Lock();
+
+            if ((SchedFlags & ThreadSchedState.LowNibbleMask) != ThreadSchedState.Paused || !WaitingSync)
+            {
+                SyncCancelled = true;
+            }
+            else if (WithholderNode != null)
+            {
+                System.Withholders.Remove(WithholderNode);
+
+                SetNewSchedFlags(ThreadSchedState.Running);
+
+                WithholderNode = null;
+
+                SyncCancelled = true;
+            }
+            else
+            {
+                SignaledObj   = null;
+                ObjSyncResult = (int)MakeError(ErrorModule.Kernel, KernelErr.Cancelled);
+
+                SetNewSchedFlags(ThreadSchedState.Running);
+
+                SyncCancelled = false;
+            }
+
+            System.CriticalSectionLock.Unlock();
+        }
+
+        public long SetCoreAndAffinityMask(int NewCore, long NewAffinityMask)
+        {
+            System.CriticalSectionLock.Lock();
+
+            bool UseOverride = AffinityOverrideCount != 0;
+
+            //The value -3 is "do not change the preferred core".
+            if (NewCore == -3)
+            {
+                NewCore = UseOverride ? PreferredCoreOverride : PreferredCore;
+
+                if ((NewAffinityMask & (1 << NewCore)) == 0)
+                {
+                    System.CriticalSectionLock.Unlock();
+
+                    return MakeError(ErrorModule.Kernel, KernelErr.InvalidMaskValue);
+                }
+            }
+
+            if (UseOverride)
+            {
+                PreferredCoreOverride = NewCore;
+                AffinityMaskOverride  = NewAffinityMask;
+            }
+            else
+            {
+                long OldAffinityMask = AffinityMask;
+
+                PreferredCore = NewCore;
+                AffinityMask  = NewAffinityMask;
+
+                if (OldAffinityMask != NewAffinityMask)
+                {
+                    int OldCore = CurrentCore;
+
+                    if (CurrentCore >= 0 && ((AffinityMask >> CurrentCore) & 1) == 0)
+                    {
+                        if (PreferredCore < 0)
+                        {
+                            CurrentCore = HighestSetCore(AffinityMask);
+                        }
+                        else
+                        {
+                            CurrentCore = PreferredCore;
+                        }
+                    }
+
+                    AdjustSchedulingForNewAffinity(OldAffinityMask, OldCore);
+                }
+            }
+
+            System.CriticalSectionLock.Unlock();
+
+            return 0;
+        }
+
+        private static int HighestSetCore(long Mask)
+        {
+            for (int Core = KScheduler.CpuCoresCount - 1; Core >= 0; Core--)
+            {
+                if (((Mask >> Core) & 1) != 0)
+                {
+                    return Core;
+                }
+            }
+
+            return -1;
+        }
+
+        private void CombineForcePauseFlags()
+        {
+            ThreadSchedState OldFlags  = SchedFlags;
+            ThreadSchedState LowNibble = SchedFlags & ThreadSchedState.LowNibbleMask;
+
+            SchedFlags = LowNibble | ForcePauseFlags;
+
+            AdjustScheduling(OldFlags);
+        }
+
+        private void SetNewSchedFlags(ThreadSchedState NewFlags)
+        {
+            System.CriticalSectionLock.Lock();
+
+            ThreadSchedState OldFlags = SchedFlags;
+
+            SchedFlags = (OldFlags & ThreadSchedState.HighNibbleMask) | NewFlags;
+
+            if ((OldFlags & ThreadSchedState.LowNibbleMask) != NewFlags)
+            {
+                AdjustScheduling(OldFlags);
+            }
+
+            System.CriticalSectionLock.Unlock();
+        }
+
+        public void ReleaseAndResume()
+        {
+            System.CriticalSectionLock.Lock();
+
+            if ((SchedFlags & ThreadSchedState.LowNibbleMask) == ThreadSchedState.Paused)
+            {
+                if (WithholderNode != null)
+                {
+                    System.Withholders.Remove(WithholderNode);
+
+                    SetNewSchedFlags(ThreadSchedState.Running);
+
+                    WithholderNode = null;
+                }
+                else
+                {
+                    SetNewSchedFlags(ThreadSchedState.Running);
+                }
+            }
+
+            System.CriticalSectionLock.Unlock();
+        }
+
+        public void Reschedule(ThreadSchedState NewFlags)
+        {
+            System.CriticalSectionLock.Lock();
+
+            ThreadSchedState OldFlags = SchedFlags;
+
+            SchedFlags = (OldFlags & ThreadSchedState.HighNibbleMask) |
+                         (NewFlags & ThreadSchedState.LowNibbleMask);
+
+            AdjustScheduling(OldFlags);
+
+            System.CriticalSectionLock.Unlock();
+        }
+
+        public void AddMutexWaiter(KThread Requester)
+        {
+            AddToMutexWaitersList(Requester);
+
+            Requester.MutexOwner = this;
+
+            UpdatePriorityInheritance();
+        }
+
+        public void RemoveMutexWaiter(KThread Thread)
+        {
+            if (Thread.MutexWaiterNode?.List != null)
+            {
+                MutexWaiters.Remove(Thread.MutexWaiterNode);
+            }
+
+            Thread.MutexOwner = null;
+
+            UpdatePriorityInheritance();
+        }
+
+        public KThread RelinquishMutex(long MutexAddress, out int Count)
+        {
+            Count = 0;
+
+            if (MutexWaiters.First == null)
+            {
+                return null;
+            }
+
+            KThread NewMutexOwner = null;
+
+            LinkedListNode<KThread> CurrentNode = MutexWaiters.First;
+
+            do
+            {
+                //Skip all threads that are not waiting for this mutex.
+                while (CurrentNode != null && CurrentNode.Value.MutexAddress != MutexAddress)
+                {
+                    CurrentNode = CurrentNode.Next;
+                }
+
+                if (CurrentNode == null)
+                {
+                    break;
+                }
+
+                LinkedListNode<KThread> NextNode = CurrentNode.Next;
+
+                MutexWaiters.Remove(CurrentNode);
+
+                CurrentNode.Value.MutexOwner = NewMutexOwner;
+
+                if (NewMutexOwner != null)
+                {
+                    //New owner was already selected, re-insert on new owner list.
+                    NewMutexOwner.AddToMutexWaitersList(CurrentNode.Value);
+                }
+                else
+                {
+                    //New owner not selected yet, use current thread.
+                    NewMutexOwner = CurrentNode.Value;
+                }
+
+                Count++;
+
+                CurrentNode = NextNode;
+            }
+            while (CurrentNode != null);
+
+            if (NewMutexOwner != null)
+            {
+                UpdatePriorityInheritance();
+
+                NewMutexOwner.UpdatePriorityInheritance();
+            }
+
+            return NewMutexOwner;
+        }
+
+        private void UpdatePriorityInheritance()
+        {
+            //If any of the threads waiting for the mutex has
+            //higher priority than the current thread, then
+            //the current thread inherits that priority.
+            int HighestPriority = BasePriority;
+
+            if (MutexWaiters.First != null)
+            {
+                int WaitingDynamicPriority = MutexWaiters.First.Value.DynamicPriority;
+
+                if (WaitingDynamicPriority < HighestPriority)
+                {
+                    HighestPriority = WaitingDynamicPriority;
+                }
+            }
+
+            if (HighestPriority != DynamicPriority)
+            {
+                int OldPriority = DynamicPriority;
+
+                DynamicPriority = HighestPriority;
+
+                AdjustSchedulingForNewPriority(OldPriority);
+
+                if (MutexOwner != null)
+                {
+                    //Remove and re-insert to ensure proper sorting based on new priority.
+                    MutexOwner.MutexWaiters.Remove(MutexWaiterNode);
+
+                    MutexOwner.AddToMutexWaitersList(this);
+
+                    MutexOwner.UpdatePriorityInheritance();
+                }
+            }
+        }
+
+        private void AddToMutexWaitersList(KThread Thread)
+        {
+            LinkedListNode<KThread> NextPrio = MutexWaiters.First;
+
+            int CurrentPriority = Thread.DynamicPriority;
+
+            while (NextPrio != null && NextPrio.Value.DynamicPriority <= CurrentPriority)
+            {
+                NextPrio = NextPrio.Next;
+            }
+
+            if (NextPrio != null)
+            {
+                Thread.MutexWaiterNode = MutexWaiters.AddBefore(NextPrio, Thread);
+            }
+            else
+            {
+                Thread.MutexWaiterNode = MutexWaiters.AddLast(Thread);
+            }
+        }
+
+        private void AdjustScheduling(ThreadSchedState OldFlags)
+        {
+            if (OldFlags == SchedFlags)
+            {
+                return;
+            }
+
+            if (OldFlags == ThreadSchedState.Running)
+            {
+                //Was running, now it's stopped.
+                if (CurrentCore >= 0)
+                {
+                    SchedulingData.Unschedule(DynamicPriority, CurrentCore, this);
+                }
+
+                for (int Core = 0; Core < KScheduler.CpuCoresCount; Core++)
+                {
+                    if (Core != CurrentCore && ((AffinityMask >> Core) & 1) != 0)
+                    {
+                        SchedulingData.Unsuggest(DynamicPriority, Core, this);
+                    }
+                }
+            }
+            else if (SchedFlags == ThreadSchedState.Running)
+            {
+                //Was stopped, now it's running.
+                if (CurrentCore >= 0)
+                {
+                    SchedulingData.Schedule(DynamicPriority, CurrentCore, this);
+                }
+
+                for (int Core = 0; Core < KScheduler.CpuCoresCount; Core++)
+                {
+                    if (Core != CurrentCore && ((AffinityMask >> Core) & 1) != 0)
+                    {
+                        SchedulingData.Suggest(DynamicPriority, Core, this);
+                    }
+                }
+            }
+
+            Scheduler.ThreadReselectionRequested = true;
+        }
+
+        private void AdjustSchedulingForNewPriority(int OldPriority)
+        {
+            if (SchedFlags != ThreadSchedState.Running)
+            {
+                return;
+            }
+
+            //Remove thread from the old priority queues.
+            if (CurrentCore >= 0)
+            {
+                SchedulingData.Unschedule(OldPriority, CurrentCore, this);
+            }
+
+            for (int Core = 0; Core < KScheduler.CpuCoresCount; Core++)
+            {
+                if (Core != CurrentCore && ((AffinityMask >> Core) & 1) != 0)
+                {
+                    SchedulingData.Unsuggest(OldPriority, Core, this);
+                }
+            }
+
+            //Add thread to the new priority queues.
+            KThread CurrentThread = Scheduler.GetCurrentThread();
+
+            if (CurrentCore >= 0)
+            {
+                if (CurrentThread == this)
+                {
+                    SchedulingData.SchedulePrepend(DynamicPriority, CurrentCore, this);
+                }
+                else
+                {
+                    SchedulingData.Schedule(DynamicPriority, CurrentCore, this);
+                }
+            }
+
+            for (int Core = 0; Core < KScheduler.CpuCoresCount; Core++)
+            {
+                if (Core != CurrentCore && ((AffinityMask >> Core) & 1) != 0)
+                {
+                    SchedulingData.Suggest(DynamicPriority, Core, this);
+                }
+            }
+
+            Scheduler.ThreadReselectionRequested = true;
+        }
+
+        private void AdjustSchedulingForNewAffinity(long OldAffinityMask, int OldCore)
+        {
+            if (SchedFlags != ThreadSchedState.Running || DynamicPriority >= KScheduler.PrioritiesCount)
+            {
+                return;
+            }
+
+            //Remove from old queues.
+            for (int Core = 0; Core < KScheduler.CpuCoresCount; Core++)
+            {
+                if (((OldAffinityMask >> Core) & 1) != 0)
+                {
+                    if (Core == OldCore)
+                    {
+                        SchedulingData.Unschedule(DynamicPriority, Core, this);
+                    }
+                    else
+                    {
+                        SchedulingData.Unsuggest(DynamicPriority, Core, this);
+                    }
+                }
+            }
+
+            //Insert on new queues.
+            for (int Core = 0; Core < KScheduler.CpuCoresCount; Core++)
+            {
+                if (((AffinityMask >> Core) & 1) != 0)
+                {
+                    if (Core == CurrentCore)
+                    {
+                        SchedulingData.Schedule(DynamicPriority, Core, this);
+                    }
+                    else
+                    {
+                        SchedulingData.Suggest(DynamicPriority, Core, this);
+                    }
+                }
+            }
+
+            Scheduler.ThreadReselectionRequested = true;
+        }
+
+        public override bool IsSignaled()
+        {
+            return HasExited;
+        }
+
+        public void ClearExclusive()
+        {
+            Owner.Memory.ClearExclusive(CurrentCore);
+        }
+
+        public void TimeUp()
+        {
+            System.CriticalSectionLock.Lock();
+
+            SetNewSchedFlags(ThreadSchedState.Running);
+
+            System.CriticalSectionLock.Unlock();
+        }
+    }
 }

Ryujinx.HLE/HOS/Kernel/KTimeManager.cs

@@ -0,0 +1,134 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading;
+
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    class KTimeManager : IDisposable
+    {
+        private class WaitingObject
+        {
+            public IKFutureSchedulerObject Object { get; private set; }
+
+            public long TimePoint { get; private set; }
+
+            public WaitingObject(IKFutureSchedulerObject Object, long TimePoint)
+            {
+                this.Object    = Object;
+                this.TimePoint = TimePoint;
+            }
+        }
+
+        private List<WaitingObject> WaitingObjects;
+
+        private AutoResetEvent WaitEvent;
+
+        private Stopwatch Counter;
+
+        private bool KeepRunning;
+
+        public KTimeManager()
+        {
+            WaitingObjects = new List<WaitingObject>();
+
+            Counter = new Stopwatch();
+
+            Counter.Start();
+
+            KeepRunning = true;
+
+            Thread Work = new Thread(WaitAndCheckScheduledObjects);
+
+            Work.Start();
+        }
+
+        public void ScheduleFutureInvocation(IKFutureSchedulerObject Object, long Timeout)
+        {
+            lock (WaitingObjects)
+            {
+                long TimePoint = Counter.ElapsedMilliseconds + ConvertNanosecondsToMilliseconds(Timeout);
+
+                WaitingObjects.Add(new WaitingObject(Object, TimePoint));
+            }
+
+            WaitEvent.Set();
+        }
+
+        private long ConvertNanosecondsToMilliseconds(long Timeout)
+        {
+            Timeout /= 1000000;
+
+            if ((ulong)Timeout > int.MaxValue)
+            {
+                return int.MaxValue;
+            }
+
+            return Timeout;
+        }
+
+        public void UnscheduleFutureInvocation(IKFutureSchedulerObject Object)
+        {
+            lock (WaitingObjects)
+            {
+                WaitingObjects.RemoveAll(x => x.Object == Object);
+            }
+        }
+
+        private void WaitAndCheckScheduledObjects()
+        {
+            using (WaitEvent = new AutoResetEvent(false))
+            {
+                while (KeepRunning)
+                {
+                    Monitor.Enter(WaitingObjects);
+
+                    WaitingObject Next = WaitingObjects.OrderBy(x => x.TimePoint).FirstOrDefault();
+
+                    Monitor.Exit(WaitingObjects);
+
+                    if (Next != null)
+                    {
+                        long TimePoint = Counter.ElapsedMilliseconds;
+
+                        if (Next.TimePoint > TimePoint)
+                        {
+                            WaitEvent.WaitOne((int)(Next.TimePoint - TimePoint));
+                        }
+
+                        Monitor.Enter(WaitingObjects);
+
+                        bool TimeUp = Counter.ElapsedMilliseconds >= Next.TimePoint && WaitingObjects.Remove(Next);
+
+                        Monitor.Exit(WaitingObjects);
+
+                        if (TimeUp)
+                        {
+                            Next.Object.TimeUp();
+                        }
+                    }
+                    else
+                    {
+                        WaitEvent.WaitOne();
+                    }
+                }
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+        }
+
+        protected virtual void Dispose(bool Disposing)
+        {
+            if (Disposing)
+            {
+                KeepRunning = false;
+
+                WaitEvent?.Set();
+            }
+        }
+    }
+}

Ryujinx.HLE/HOS/Kernel/KernelErr.cs

@@ -2,6 +2,7 @@ namespace Ryujinx.HLE.HOS.Kernel
 {
     static class KernelErr
     {
+        public const int ThreadTerminating = 59;
         public const int InvalidSize       = 101;
         public const int InvalidAddress    = 102;
         public const int OutOfMemory       = 104;
@@ -13,7 +14,7 @@ namespace Ryujinx.HLE.HOS.Kernel
         public const int InvalidHandle     = 114;
         public const int InvalidMaskValue  = 116;
         public const int Timeout           = 117;
-        public const int Canceled          = 118;
+        public const int Cancelled         = 118;
         public const int CountOutOfRange   = 119;
         public const int InvalidEnumValue  = 120;
         public const int InvalidThread     = 122;

Ryujinx.HLE/HOS/Kernel/NsTimeConverter.cs

@@ -1,19 +0,0 @@
-namespace Ryujinx.HLE.HOS.Kernel
-{
-    static class NsTimeConverter
-    {
-        public static int GetTimeMs(ulong Ns)
-        {
-            ulong Ms = Ns / 1_000_000;
-
-            if (Ms < int.MaxValue)
-            {
-                return (int)Ms;
-            }
-            else
-            {
-                return int.MaxValue;
-            }
-        }
-    }
-}

Ryujinx.HLE/HOS/Kernel/SignalType.cs

@@ -0,0 +1,9 @@
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    enum SignalType
+    {
+        Signal                    = 0,
+        SignalAndIncrementIfEqual = 1,
+        SignalAndModifyIfEqual    = 2
+    }
+}

Ryujinx.HLE/HOS/Kernel/SvcHandler.cs

@@ -1,11 +1,10 @@
 using ChocolArm64.Events;
 using ChocolArm64.Memory;
 using ChocolArm64.State;
+using Ryujinx.HLE.HOS.Ipc;
 using Ryujinx.HLE.Logging;
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.Threading;
 
 namespace Ryujinx.HLE.HOS.Kernel
 {
@@ -17,9 +16,28 @@ namespace Ryujinx.HLE.HOS.Kernel
 
         private Switch  Device;
         private Process Process;
+        private Horizon System;
         private AMemory Memory;
 
-        private ConcurrentDictionary<KThread, AutoResetEvent> SyncWaits;
+        private struct HleIpcMessage
+        {
+            public KThread    Thread     { get; private set; }
+            public KSession   Session    { get; private set; }
+            public IpcMessage Message    { get; private set; }
+            public long       MessagePtr { get; private set; }
+
+            public HleIpcMessage(
+                KThread    Thread,
+                KSession   Session,
+                IpcMessage Message,
+                long       MessagePtr)
+            {
+                this.Thread     = Thread;
+                this.Session    = Session;
+                this.Message    = Message;
+                this.MessagePtr = MessagePtr;
+            }
+        }
 
         private const uint SelfThreadHandle  = 0xffff8000;
         private const uint SelfProcessHandle = 0xffff8001;
@@ -69,14 +87,14 @@ namespace Ryujinx.HLE.HOS.Kernel
                 { 0x2d, SvcUnmapPhysicalMemory           },
                 { 0x32, SvcSetThreadActivity             },
                 { 0x33, SvcGetThreadContext3             },
-                { 0x34, SvcWaitForAddress                }
+                { 0x34, SvcWaitForAddress                },
+                { 0x35, SvcSignalToAddress               }
             };
 
             this.Device  = Device;
             this.Process = Process;
+            this.System  = Process.Device.System;
             this.Memory  = Process.Memory;
-
-            SyncWaits = new ConcurrentDictionary<KThread, AutoResetEvent>();
         }
 
         static SvcHandler()
@@ -96,8 +114,6 @@ namespace Ryujinx.HLE.HOS.Kernel
 
                 Func(ThreadState);
 
-                Process.Scheduler.Reschedule(Process.GetThread(ThreadState.Tpidr));
-
                 Device.Log.PrintDebug(LogClass.KernelSvc, $"{Func.Method.Name} ended.");
             }
             else

Ryujinx.HLE/HOS/Kernel/SvcSystem.cs

@@ -68,7 +68,7 @@ namespace Ryujinx.HLE.HOS.Kernel
 
             if (Event != null)
             {
-                Event.WaitEvent.Reset();
+                Event.Reset();
 
                 ThreadState.X0 = 0;
             }
@@ -80,115 +80,6 @@ namespace Ryujinx.HLE.HOS.Kernel
             }
         }
 
-        private void SvcWaitSynchronization(AThreadState ThreadState)
-        {
-            long  HandlesPtr   = (long)ThreadState.X1;
-            int   HandlesCount =  (int)ThreadState.X2;
-            ulong Timeout      =       ThreadState.X3;
-
-            Device.Log.PrintDebug(LogClass.KernelSvc,
-                "HandlesPtr = 0x"   + HandlesPtr  .ToString("x16") + ", " +
-                "HandlesCount = 0x" + HandlesCount.ToString("x8")  + ", " +
-                "Timeout = 0x"      + Timeout     .ToString("x16"));
-
-            if ((uint)HandlesCount > 0x40)
-            {
-                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.CountOutOfRange);
-
-                return;
-            }
-
-            KThread CurrThread = Process.GetThread(ThreadState.Tpidr);
-
-            WaitHandle[] Handles = new WaitHandle[HandlesCount + 1];
-
-            for (int Index = 0; Index < HandlesCount; Index++)
-            {
-                int Handle = Memory.ReadInt32(HandlesPtr + Index * 4);
-
-                KSynchronizationObject SyncObj = Process.HandleTable.GetData<KSynchronizationObject>(Handle);
-
-                if (SyncObj == null)
-                {
-                    Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid handle 0x{Handle:x8}!");
-
-                    ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
-
-                    return;
-                }
-
-                Handles[Index] = SyncObj.WaitEvent;
-            }
-
-            using (AutoResetEvent WaitEvent = new AutoResetEvent(false))
-            {
-                if (!SyncWaits.TryAdd(CurrThread, WaitEvent))
-                {
-                    throw new InvalidOperationException();
-                }
-
-                Handles[HandlesCount] = WaitEvent;
-
-                Process.Scheduler.Suspend(CurrThread);
-
-                int HandleIndex;
-
-                ulong Result = 0;
-
-                if (Timeout != ulong.MaxValue)
-                {
-                    HandleIndex = WaitHandle.WaitAny(Handles, NsTimeConverter.GetTimeMs(Timeout));
-                }
-                else
-                {
-                    HandleIndex = WaitHandle.WaitAny(Handles);
-                }
-
-                if (HandleIndex == WaitHandle.WaitTimeout)
-                {
-                    Result = MakeError(ErrorModule.Kernel, KernelErr.Timeout);
-                }
-                else if (HandleIndex == HandlesCount)
-                {
-                    Result = MakeError(ErrorModule.Kernel, KernelErr.Canceled);
-                }
-
-                SyncWaits.TryRemove(CurrThread, out _);
-
-                Process.Scheduler.Resume(CurrThread);
-
-                ThreadState.X0 = Result;
-
-                if (Result == 0)
-                {
-                    ThreadState.X1 = (ulong)HandleIndex;
-                }
-            }
-        }
-
-        private void SvcCancelSynchronization(AThreadState ThreadState)
-        {
-            int ThreadHandle = (int)ThreadState.X0;
-
-            KThread Thread = GetThread(ThreadState.Tpidr, ThreadHandle);
-
-            if (Thread == null)
-            {
-                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid thread handle 0x{ThreadHandle:x8}!");
-
-                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
-
-                return;
-            }
-
-            if (SyncWaits.TryRemove(Thread, out AutoResetEvent WaitEvent))
-            {
-                WaitEvent.Set();
-            }
-
-            ThreadState.X0 = 0;
-        }
-
         private void SvcGetSystemTick(AThreadState ThreadState)
         {
             ThreadState.X0 = ThreadState.CntpctEl0;
@@ -203,7 +94,7 @@ namespace Ryujinx.HLE.HOS.Kernel
 
             //TODO: Validate that app has perms to access the service, and that the service
             //actually exists, return error codes otherwise.
-            KSession Session = new KSession(ServiceFactory.MakeService(Name), Name);
+            KSession Session = new KSession(ServiceFactory.MakeService(System, Name), Name);
 
             ulong Handle = (ulong)Process.HandleTable.OpenHandle(Session);
 
@@ -225,27 +116,38 @@ namespace Ryujinx.HLE.HOS.Kernel
                  (int)ThreadState.X2);
         }
 
-        private void SendSyncRequest(AThreadState ThreadState, long CmdPtr, long Size, int Handle)
+        private void SendSyncRequest(AThreadState ThreadState, long MessagePtr, long Size, int Handle)
         {
             KThread CurrThread = Process.GetThread(ThreadState.Tpidr);
 
-            byte[] CmdData = Memory.ReadBytes(CmdPtr, Size);
+            byte[] MessageData = Memory.ReadBytes(MessagePtr, Size);
 
             KSession Session = Process.HandleTable.GetData<KSession>(Handle);
 
             if (Session != null)
             {
-                Process.Scheduler.Suspend(CurrThread);
+                //Process.Scheduler.Suspend(CurrThread);
 
-                IpcMessage Cmd = new IpcMessage(CmdData, CmdPtr);
+                System.CriticalSectionLock.Lock();
 
-                long Result = IpcHandler.IpcCall(Device, Process, Memory, Session, Cmd, CmdPtr);
+                KThread CurrentThread = System.Scheduler.GetCurrentThread();
 
-                Thread.Yield();
+                CurrentThread.SignaledObj   = null;
+                CurrentThread.ObjSyncResult = 0;
 
-                Process.Scheduler.Resume(CurrThread);
+                CurrentThread.Reschedule(ThreadSchedState.Paused);
 
-                ThreadState.X0 = (ulong)Result;
+                IpcMessage Message = new IpcMessage(MessageData, MessagePtr);
+
+                ThreadPool.QueueUserWorkItem(ProcessIpcRequest, new HleIpcMessage(
+                    CurrentThread,
+                    Session,
+                    Message,
+                    MessagePtr));
+
+                System.CriticalSectionLock.Unlock();
+
+                ThreadState.X0 = (ulong)CurrentThread.ObjSyncResult;
             }
             else
             {
@@ -255,6 +157,21 @@ namespace Ryujinx.HLE.HOS.Kernel
             }
         }
 
+        private void ProcessIpcRequest(object State)
+        {
+            HleIpcMessage IpcMessage = (HleIpcMessage)State;
+
+            IpcMessage.Thread.ObjSyncResult = (int)IpcHandler.IpcCall(
+                Device,
+                Process,
+                Memory,
+                IpcMessage.Session,
+                IpcMessage.Message,
+                IpcMessage.MessagePtr);
+
+            IpcMessage.Thread.Reschedule(ThreadSchedState.Running);
+        }
+
         private void SvcBreak(AThreadState ThreadState)
         {
             long Reason  = (long)ThreadState.X0;

Ryujinx.HLE/HOS/Kernel/SvcThread.cs

@@ -1,6 +1,5 @@
 using ChocolArm64.State;
 using Ryujinx.HLE.Logging;
-using System.Threading;
 
 using static Ryujinx.HLE.HOS.ErrorCode;
 
@@ -54,14 +53,18 @@ namespace Ryujinx.HLE.HOS.Kernel
         {
             int Handle = (int)ThreadState.X0;
 
-            KThread NewThread = Process.HandleTable.GetData<KThread>(Handle);
+            KThread Thread = Process.HandleTable.GetData<KThread>(Handle);
 
-            if (NewThread != null)
+            if (Thread != null)
             {
-                Process.Scheduler.StartThread(NewThread);
-                Process.Scheduler.SetReschedule(NewThread.ProcessorId);
+                long Result = Thread.Start();
 
-                ThreadState.X0 = 0;
+                if (Result != 0)
+                {
+                    Device.Log.PrintWarning(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
+                }
+
+                ThreadState.X0 = (ulong)Result;
             }
             else
             {
@@ -73,30 +76,37 @@ namespace Ryujinx.HLE.HOS.Kernel
 
         private void SvcExitThread(AThreadState ThreadState)
         {
-            KThread CurrThread = Process.GetThread(ThreadState.Tpidr);
+            KThread CurrentThread = System.Scheduler.GetCurrentThread();
 
-            CurrThread.Thread.StopExecution();
+            CurrentThread.Exit();
+
+            System.Scheduler.StopThread(CurrentThread);
+
+            System.Scheduler.CoreContexts[CurrentThread.CurrentCore].RemoveThread(CurrentThread);
         }
 
         private void SvcSleepThread(AThreadState ThreadState)
         {
-            ulong TimeoutNs = ThreadState.X0;
+            long Timeout = (long)ThreadState.X0;
 
-            Device.Log.PrintDebug(LogClass.KernelSvc, "Timeout = 0x" + TimeoutNs.ToString("x16"));
+            Device.Log.PrintDebug(LogClass.KernelSvc, "Timeout = 0x" + Timeout.ToString("x16"));
 
-            KThread CurrThread = Process.GetThread(ThreadState.Tpidr);
+            KThread CurrentThread = System.Scheduler.GetCurrentThread();
 
-            if (TimeoutNs == 0 || TimeoutNs == ulong.MaxValue)
+            if (Timeout < 1)
             {
-                Process.Scheduler.Yield(CurrThread);
+                switch (Timeout)
+                {
+                    case  0: CurrentThread.Yield();                        break;
+                    case -1: CurrentThread.YieldWithLoadBalancing();       break;
+                    case -2: CurrentThread.YieldAndWaitForLoadBalancing(); break;
+                }
             }
             else
             {
-                Process.Scheduler.Suspend(CurrThread);
+                CurrentThread.Sleep(Timeout);
 
-                Thread.Sleep(NsTimeConverter.GetTimeMs(TimeoutNs));
-
-                Process.Scheduler.Resume(CurrThread);
+                ThreadState.X0 = 0;
             }
         }
 
@@ -109,7 +119,7 @@ namespace Ryujinx.HLE.HOS.Kernel
             if (Thread != null)
             {
                 ThreadState.X0 = 0;
-                ThreadState.X1 = (ulong)Thread.ActualPriority;
+                ThreadState.X1 = (ulong)Thread.DynamicPriority;
             }
             else
             {
@@ -128,20 +138,22 @@ namespace Ryujinx.HLE.HOS.Kernel
                 "Handle = 0x"   + Handle  .ToString("x8") + ", " +
                 "Priority = 0x" + Priority.ToString("x8"));
 
+            //TODO: NPDM check.
+
             KThread Thread = GetThread(ThreadState.Tpidr, Handle);
 
-            if (Thread != null)
-            {
-                Thread.SetPriority(Priority);
-
-                ThreadState.X0 = 0;
-            }
-            else
+            if (Thread == null)
             {
                 Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid thread handle 0x{Handle:x8}!");
 
                 ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+
+                return;
             }
+
+            Thread.SetPriority(Priority);
+
+            ThreadState.X0 = 0;
         }
 
         private void SvcGetThreadCoreMask(AThreadState ThreadState)
@@ -155,8 +167,8 @@ namespace Ryujinx.HLE.HOS.Kernel
             if (Thread != null)
             {
                 ThreadState.X0 = 0;
-                ThreadState.X1 = (ulong)Thread.IdealCore;
-                ThreadState.X2 = (ulong)Thread.CoreMask;
+                ThreadState.X1 = (ulong)Thread.PreferredCore;
+                ThreadState.X2 = (ulong)Thread.AffinityMask;
             }
             else
             {
@@ -168,40 +180,40 @@ namespace Ryujinx.HLE.HOS.Kernel
 
         private void SvcSetThreadCoreMask(AThreadState ThreadState)
         {
-            int  Handle    =  (int)ThreadState.X0;
-            int  IdealCore =  (int)ThreadState.X1;
-            long CoreMask  = (long)ThreadState.X2;
+            int  ThreadHandle  =  (int)ThreadState.X0;
+            int  PrefferedCore =  (int)ThreadState.X1;
+            long AffinityMask  = (long)ThreadState.X2;
 
             Device.Log.PrintDebug(LogClass.KernelSvc,
-                "Handle = 0x"    + Handle   .ToString("x8") + ", " +
-                "IdealCore = 0x" + IdealCore.ToString("x8") + ", " +
-                "CoreMask = 0x"  + CoreMask .ToString("x16"));
+                "ThreadHandle = 0x"  + ThreadHandle .ToString("x8") + ", " +
+                "PrefferedCore = 0x" + PrefferedCore.ToString("x8") + ", " +
+                "AffinityMask = 0x"  + AffinityMask .ToString("x16"));
 
-            KThread Thread = GetThread(ThreadState.Tpidr, Handle);
-
-            if (IdealCore == -2)
+            if (PrefferedCore == -2)
             {
                 //TODO: Get this value from the NPDM file.
-                IdealCore = 0;
+                PrefferedCore = 0;
 
-                CoreMask = 1 << IdealCore;
+                AffinityMask = 1 << PrefferedCore;
             }
             else
             {
-                if ((uint)IdealCore > 3)
+                //TODO: Check allowed cores from NPDM file.
+
+                if ((uint)PrefferedCore > 3)
                 {
-                    if ((IdealCore | 2) != -1)
+                    if ((PrefferedCore | 2) != -1)
                     {
-                        Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid core id 0x{IdealCore:x8}!");
+                        Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid core id 0x{PrefferedCore:x8}!");
 
                         ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidCoreId);
 
                         return;
                     }
                 }
-                else if ((CoreMask & (1 << IdealCore)) == 0)
+                else if ((AffinityMask & (1 << PrefferedCore)) == 0)
                 {
-                    Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid core mask 0x{CoreMask:x8}!");
+                    Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid core mask 0x{AffinityMask:x8}!");
 
                     ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidMaskValue);
 
@@ -209,35 +221,30 @@ namespace Ryujinx.HLE.HOS.Kernel
                 }
             }
 
+            KThread Thread = GetThread(ThreadState.Tpidr, ThreadHandle);
+
             if (Thread == null)
             {
-                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid thread handle 0x{Handle:x8}!");
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid thread handle 0x{ThreadHandle:x8}!");
 
                 ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
 
                 return;
             }
 
-            //-1 is used as "don't care", so the IdealCore value is ignored.
-            //-2 is used as "use NPDM default core id" (handled above).
-            //-3 is used as "don't update", the old IdealCore value is kept.
-            if (IdealCore == -3 && (CoreMask & (1 << Thread.IdealCore)) == 0)
+            long Result = Thread.SetCoreAndAffinityMask(PrefferedCore, AffinityMask);
+
+            if (Result != 0)
             {
-                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid core mask 0x{CoreMask:x8}!");
-
-                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidMaskValue);
-
-                return;
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
             }
 
-            Process.Scheduler.ChangeCore(Thread, IdealCore, (int)CoreMask);
-
-            ThreadState.X0 = 0;
+            ThreadState.X0 = (ulong)Result;
         }
 
         private void SvcGetCurrentProcessorNumber(AThreadState ThreadState)
         {
-            ThreadState.X0 = (ulong)Process.GetThread(ThreadState.Tpidr).ActualCore;
+            ThreadState.X0 = (ulong)Process.GetThread(ThreadState.Tpidr).CurrentCore;
         }
 
         private void SvcGetThreadId(AThreadState ThreadState)
@@ -262,22 +269,36 @@ namespace Ryujinx.HLE.HOS.Kernel
         private void SvcSetThreadActivity(AThreadState ThreadState)
         {
             int  Handle = (int)ThreadState.X0;
-            bool Active = (int)ThreadState.X1 == 0;
+            bool Pause  = (int)ThreadState.X1 == 1;
 
             KThread Thread = Process.HandleTable.GetData<KThread>(Handle);
 
-            if (Thread != null)
-            {
-                Process.Scheduler.SetThreadActivity(Thread, Active);
-
-                ThreadState.X0 = 0;
-            }
-            else
+            if (Thread == null)
             {
                 Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid thread handle 0x{Handle:x8}!");
 
                 ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+
+                return;
             }
+
+            if (Thread.Owner != Process)
+            {
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid thread owner process!");
+
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+
+                return;
+            }
+
+            long Result = Thread.SetActivity(Pause);
+
+            if (Result != 0)
+            {
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
+            }
+
+            ThreadState.X0 = (ulong)Result;
         }
 
         private void SvcGetThreadContext3(AThreadState ThreadState)
@@ -305,79 +326,79 @@ namespace Ryujinx.HLE.HOS.Kernel
                 return;
             }
 
-            Memory.WriteUInt64(Position + 0x0,  ThreadState.X0);
-            Memory.WriteUInt64(Position + 0x8,  ThreadState.X1);
-            Memory.WriteUInt64(Position + 0x10, ThreadState.X2);
-            Memory.WriteUInt64(Position + 0x18, ThreadState.X3);
-            Memory.WriteUInt64(Position + 0x20, ThreadState.X4);
-            Memory.WriteUInt64(Position + 0x28, ThreadState.X5);
-            Memory.WriteUInt64(Position + 0x30, ThreadState.X6);
-            Memory.WriteUInt64(Position + 0x38, ThreadState.X7);
-            Memory.WriteUInt64(Position + 0x40, ThreadState.X8);
-            Memory.WriteUInt64(Position + 0x48, ThreadState.X9);
-            Memory.WriteUInt64(Position + 0x50, ThreadState.X10);
-            Memory.WriteUInt64(Position + 0x58, ThreadState.X11);
-            Memory.WriteUInt64(Position + 0x60, ThreadState.X12);
-            Memory.WriteUInt64(Position + 0x68, ThreadState.X13);
-            Memory.WriteUInt64(Position + 0x70, ThreadState.X14);
-            Memory.WriteUInt64(Position + 0x78, ThreadState.X15);
-            Memory.WriteUInt64(Position + 0x80, ThreadState.X16);
-            Memory.WriteUInt64(Position + 0x88, ThreadState.X17);
-            Memory.WriteUInt64(Position + 0x90, ThreadState.X18);
-            Memory.WriteUInt64(Position + 0x98, ThreadState.X19);
-            Memory.WriteUInt64(Position + 0xa0, ThreadState.X20);
-            Memory.WriteUInt64(Position + 0xa8, ThreadState.X21);
-            Memory.WriteUInt64(Position + 0xb0, ThreadState.X22);
-            Memory.WriteUInt64(Position + 0xb8, ThreadState.X23);
-            Memory.WriteUInt64(Position + 0xc0, ThreadState.X24);
-            Memory.WriteUInt64(Position + 0xc8, ThreadState.X25);
-            Memory.WriteUInt64(Position + 0xd0, ThreadState.X26);
-            Memory.WriteUInt64(Position + 0xd8, ThreadState.X27);
-            Memory.WriteUInt64(Position + 0xe0, ThreadState.X28);
-            Memory.WriteUInt64(Position + 0xe8, ThreadState.X29);
-            Memory.WriteUInt64(Position + 0xf0, ThreadState.X30);
-            Memory.WriteUInt64(Position + 0xf8, ThreadState.X31);
+            Memory.WriteUInt64(Position + 0x0,  Thread.Context.ThreadState.X0);
+            Memory.WriteUInt64(Position + 0x8,  Thread.Context.ThreadState.X1);
+            Memory.WriteUInt64(Position + 0x10, Thread.Context.ThreadState.X2);
+            Memory.WriteUInt64(Position + 0x18, Thread.Context.ThreadState.X3);
+            Memory.WriteUInt64(Position + 0x20, Thread.Context.ThreadState.X4);
+            Memory.WriteUInt64(Position + 0x28, Thread.Context.ThreadState.X5);
+            Memory.WriteUInt64(Position + 0x30, Thread.Context.ThreadState.X6);
+            Memory.WriteUInt64(Position + 0x38, Thread.Context.ThreadState.X7);
+            Memory.WriteUInt64(Position + 0x40, Thread.Context.ThreadState.X8);
+            Memory.WriteUInt64(Position + 0x48, Thread.Context.ThreadState.X9);
+            Memory.WriteUInt64(Position + 0x50, Thread.Context.ThreadState.X10);
+            Memory.WriteUInt64(Position + 0x58, Thread.Context.ThreadState.X11);
+            Memory.WriteUInt64(Position + 0x60, Thread.Context.ThreadState.X12);
+            Memory.WriteUInt64(Position + 0x68, Thread.Context.ThreadState.X13);
+            Memory.WriteUInt64(Position + 0x70, Thread.Context.ThreadState.X14);
+            Memory.WriteUInt64(Position + 0x78, Thread.Context.ThreadState.X15);
+            Memory.WriteUInt64(Position + 0x80, Thread.Context.ThreadState.X16);
+            Memory.WriteUInt64(Position + 0x88, Thread.Context.ThreadState.X17);
+            Memory.WriteUInt64(Position + 0x90, Thread.Context.ThreadState.X18);
+            Memory.WriteUInt64(Position + 0x98, Thread.Context.ThreadState.X19);
+            Memory.WriteUInt64(Position + 0xa0, Thread.Context.ThreadState.X20);
+            Memory.WriteUInt64(Position + 0xa8, Thread.Context.ThreadState.X21);
+            Memory.WriteUInt64(Position + 0xb0, Thread.Context.ThreadState.X22);
+            Memory.WriteUInt64(Position + 0xb8, Thread.Context.ThreadState.X23);
+            Memory.WriteUInt64(Position + 0xc0, Thread.Context.ThreadState.X24);
+            Memory.WriteUInt64(Position + 0xc8, Thread.Context.ThreadState.X25);
+            Memory.WriteUInt64(Position + 0xd0, Thread.Context.ThreadState.X26);
+            Memory.WriteUInt64(Position + 0xd8, Thread.Context.ThreadState.X27);
+            Memory.WriteUInt64(Position + 0xe0, Thread.Context.ThreadState.X28);
+            Memory.WriteUInt64(Position + 0xe8, Thread.Context.ThreadState.X29);
+            Memory.WriteUInt64(Position + 0xf0, Thread.Context.ThreadState.X30);
+            Memory.WriteUInt64(Position + 0xf8, Thread.Context.ThreadState.X31);
 
             Memory.WriteInt64(Position + 0x100, Thread.LastPc);
 
-            Memory.WriteUInt64(Position + 0x108, (ulong)ThreadState.Psr);
+            Memory.WriteUInt64(Position + 0x108, (ulong)Thread.Context.ThreadState.Psr);
 
-            Memory.WriteVector128(Position + 0x110, ThreadState.V0);
-            Memory.WriteVector128(Position + 0x120, ThreadState.V1);
-            Memory.WriteVector128(Position + 0x130, ThreadState.V2);
-            Memory.WriteVector128(Position + 0x140, ThreadState.V3);
-            Memory.WriteVector128(Position + 0x150, ThreadState.V4);
-            Memory.WriteVector128(Position + 0x160, ThreadState.V5);
-            Memory.WriteVector128(Position + 0x170, ThreadState.V6);
-            Memory.WriteVector128(Position + 0x180, ThreadState.V7);
-            Memory.WriteVector128(Position + 0x190, ThreadState.V8);
-            Memory.WriteVector128(Position + 0x1a0, ThreadState.V9);
-            Memory.WriteVector128(Position + 0x1b0, ThreadState.V10);
-            Memory.WriteVector128(Position + 0x1c0, ThreadState.V11);
-            Memory.WriteVector128(Position + 0x1d0, ThreadState.V12);
-            Memory.WriteVector128(Position + 0x1e0, ThreadState.V13);
-            Memory.WriteVector128(Position + 0x1f0, ThreadState.V14);
-            Memory.WriteVector128(Position + 0x200, ThreadState.V15);
-            Memory.WriteVector128(Position + 0x210, ThreadState.V16);
-            Memory.WriteVector128(Position + 0x220, ThreadState.V17);
-            Memory.WriteVector128(Position + 0x230, ThreadState.V18);
-            Memory.WriteVector128(Position + 0x240, ThreadState.V19);
-            Memory.WriteVector128(Position + 0x250, ThreadState.V20);
-            Memory.WriteVector128(Position + 0x260, ThreadState.V21);
-            Memory.WriteVector128(Position + 0x270, ThreadState.V22);
-            Memory.WriteVector128(Position + 0x280, ThreadState.V23);
-            Memory.WriteVector128(Position + 0x290, ThreadState.V24);
-            Memory.WriteVector128(Position + 0x2a0, ThreadState.V25);
-            Memory.WriteVector128(Position + 0x2b0, ThreadState.V26);
-            Memory.WriteVector128(Position + 0x2c0, ThreadState.V27);
-            Memory.WriteVector128(Position + 0x2d0, ThreadState.V28);
-            Memory.WriteVector128(Position + 0x2e0, ThreadState.V29);
-            Memory.WriteVector128(Position + 0x2f0, ThreadState.V30);
-            Memory.WriteVector128(Position + 0x300, ThreadState.V31);
+            Memory.WriteVector128(Position + 0x110, Thread.Context.ThreadState.V0);
+            Memory.WriteVector128(Position + 0x120, Thread.Context.ThreadState.V1);
+            Memory.WriteVector128(Position + 0x130, Thread.Context.ThreadState.V2);
+            Memory.WriteVector128(Position + 0x140, Thread.Context.ThreadState.V3);
+            Memory.WriteVector128(Position + 0x150, Thread.Context.ThreadState.V4);
+            Memory.WriteVector128(Position + 0x160, Thread.Context.ThreadState.V5);
+            Memory.WriteVector128(Position + 0x170, Thread.Context.ThreadState.V6);
+            Memory.WriteVector128(Position + 0x180, Thread.Context.ThreadState.V7);
+            Memory.WriteVector128(Position + 0x190, Thread.Context.ThreadState.V8);
+            Memory.WriteVector128(Position + 0x1a0, Thread.Context.ThreadState.V9);
+            Memory.WriteVector128(Position + 0x1b0, Thread.Context.ThreadState.V10);
+            Memory.WriteVector128(Position + 0x1c0, Thread.Context.ThreadState.V11);
+            Memory.WriteVector128(Position + 0x1d0, Thread.Context.ThreadState.V12);
+            Memory.WriteVector128(Position + 0x1e0, Thread.Context.ThreadState.V13);
+            Memory.WriteVector128(Position + 0x1f0, Thread.Context.ThreadState.V14);
+            Memory.WriteVector128(Position + 0x200, Thread.Context.ThreadState.V15);
+            Memory.WriteVector128(Position + 0x210, Thread.Context.ThreadState.V16);
+            Memory.WriteVector128(Position + 0x220, Thread.Context.ThreadState.V17);
+            Memory.WriteVector128(Position + 0x230, Thread.Context.ThreadState.V18);
+            Memory.WriteVector128(Position + 0x240, Thread.Context.ThreadState.V19);
+            Memory.WriteVector128(Position + 0x250, Thread.Context.ThreadState.V20);
+            Memory.WriteVector128(Position + 0x260, Thread.Context.ThreadState.V21);
+            Memory.WriteVector128(Position + 0x270, Thread.Context.ThreadState.V22);
+            Memory.WriteVector128(Position + 0x280, Thread.Context.ThreadState.V23);
+            Memory.WriteVector128(Position + 0x290, Thread.Context.ThreadState.V24);
+            Memory.WriteVector128(Position + 0x2a0, Thread.Context.ThreadState.V25);
+            Memory.WriteVector128(Position + 0x2b0, Thread.Context.ThreadState.V26);
+            Memory.WriteVector128(Position + 0x2c0, Thread.Context.ThreadState.V27);
+            Memory.WriteVector128(Position + 0x2d0, Thread.Context.ThreadState.V28);
+            Memory.WriteVector128(Position + 0x2e0, Thread.Context.ThreadState.V29);
+            Memory.WriteVector128(Position + 0x2f0, Thread.Context.ThreadState.V30);
+            Memory.WriteVector128(Position + 0x300, Thread.Context.ThreadState.V31);
 
-            Memory.WriteInt32(Position + 0x310, ThreadState.Fpcr);
-            Memory.WriteInt32(Position + 0x314, ThreadState.Fpsr);
-            Memory.WriteInt64(Position + 0x318, ThreadState.Tpidr);
+            Memory.WriteInt32(Position + 0x310, Thread.Context.ThreadState.Fpcr);
+            Memory.WriteInt32(Position + 0x314, Thread.Context.ThreadState.Fpsr);
+            Memory.WriteInt64(Position + 0x318, Thread.Context.ThreadState.Tpidr);
 
             ThreadState.X0 = 0;
         }

Ryujinx.HLE/HOS/Kernel/SvcThreadSync.cs

@@ -1,6 +1,5 @@
 using ChocolArm64.State;
 using Ryujinx.HLE.Logging;
-using System;
 
 using static Ryujinx.HLE.HOS.ErrorCode;
 
@@ -8,18 +7,90 @@ namespace Ryujinx.HLE.HOS.Kernel
 {
     partial class SvcHandler
     {
-        private const int MutexHasListenersMask = 0x40000000;
+        private void SvcWaitSynchronization(AThreadState ThreadState)
+        {
+            long HandlesPtr   = (long)ThreadState.X1;
+            int  HandlesCount =  (int)ThreadState.X2;
+            long Timeout      = (long)ThreadState.X3;
+
+            Device.Log.PrintDebug(LogClass.KernelSvc,
+                "HandlesPtr = 0x"   + HandlesPtr  .ToString("x16") + ", " +
+                "HandlesCount = 0x" + HandlesCount.ToString("x8")  + ", " +
+                "Timeout = 0x"      + Timeout     .ToString("x16"));
+
+            if ((uint)HandlesCount > 0x40)
+            {
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.CountOutOfRange);
+
+                return;
+            }
+
+            KSynchronizationObject[] SyncObjs = new KSynchronizationObject[HandlesCount];
+
+            for (int Index = 0; Index < HandlesCount; Index++)
+            {
+                int Handle = Memory.ReadInt32(HandlesPtr + Index * 4);
+
+                KSynchronizationObject SyncObj = Process.HandleTable.GetData<KSynchronizationObject>(Handle);
+
+                SyncObjs[Index] = SyncObj;
+            }
+
+            int HndIndex = (int)ThreadState.X1;
+
+            ulong High = ThreadState.X1 & (0xffffffffUL << 32);
+
+            long Result = System.Synchronization.WaitFor(SyncObjs, Timeout, ref HndIndex);
+
+            if (Result != 0)
+            {
+                if (Result == MakeError(ErrorModule.Kernel, KernelErr.Timeout) ||
+                    Result == MakeError(ErrorModule.Kernel, KernelErr.Cancelled))
+                {
+                    Device.Log.PrintDebug(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
+                }
+                else
+                {
+                    Device.Log.PrintWarning(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
+                }
+            }
+
+            ThreadState.X0 = (ulong)Result;
+            ThreadState.X1 = (uint)HndIndex | High;
+        }
+
+        private void SvcCancelSynchronization(AThreadState ThreadState)
+        {
+            int ThreadHandle = (int)ThreadState.X0;
+
+            Device.Log.PrintDebug(LogClass.KernelSvc, "ThreadHandle = 0x" + ThreadHandle.ToString("x8"));
+
+            KThread Thread = Process.HandleTable.GetData<KThread>(ThreadHandle);
+
+            if (Thread == null)
+            {
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid thread handle 0x{ThreadHandle:x8}!");
+
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+
+                return;
+            }
+
+            Thread.CancelSynchronization();
+
+            ThreadState.X0 = 0;
+        }
 
         private void SvcArbitrateLock(AThreadState ThreadState)
         {
-            int  OwnerThreadHandle =  (int)ThreadState.X0;
+            int  OwnerHandle     =  (int)ThreadState.X0;
             long MutexAddress    = (long)ThreadState.X1;
-            int  WaitThreadHandle  =  (int)ThreadState.X2;
+            int  RequesterHandle =  (int)ThreadState.X2;
 
             Device.Log.PrintDebug(LogClass.KernelSvc,
-                "OwnerThreadHandle = 0x" + OwnerThreadHandle.ToString("x8")  + ", " +
+                "OwnerHandle = 0x"     + OwnerHandle    .ToString("x8")  + ", " +
                 "MutexAddress = 0x"    + MutexAddress   .ToString("x16") + ", " +
-                "WaitThreadHandle = 0x"  + WaitThreadHandle .ToString("x8"));
+                "RequesterHandle = 0x" + RequesterHandle.ToString("x8"));
 
             if (IsPointingInsideKernel(MutexAddress))
             {
@@ -39,33 +110,19 @@ namespace Ryujinx.HLE.HOS.Kernel
                 return;
             }
 
-            KThread OwnerThread = Process.HandleTable.GetData<KThread>(OwnerThreadHandle);
+            long Result = System.AddressArbiter.ArbitrateLock(
+                Process,
+                Memory,
+                OwnerHandle,
+                MutexAddress,
+                RequesterHandle);
 
-            if (OwnerThread == null)
+            if (Result != 0)
             {
-                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid owner thread handle 0x{OwnerThreadHandle:x8}!");
-
-                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
-
-                return;
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
             }
 
-            KThread WaitThread = Process.HandleTable.GetData<KThread>(WaitThreadHandle);
-
-            if (WaitThread == null)
-            {
-                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid requesting thread handle 0x{WaitThreadHandle:x8}!");
-
-                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
-
-                return;
-            }
-
-            KThread CurrThread = Process.GetThread(ThreadState.Tpidr);
-
-            MutexLock(CurrThread, WaitThread, OwnerThreadHandle, WaitThreadHandle, MutexAddress);
-
-            ThreadState.X0 = 0;
+            ThreadState.X0 = (ulong)Result;
         }
 
         private void SvcArbitrateUnlock(AThreadState ThreadState)
@@ -92,9 +149,14 @@ namespace Ryujinx.HLE.HOS.Kernel
                 return;
             }
 
-            MutexUnlock(Process.GetThread(ThreadState.Tpidr), MutexAddress);
+            long Result = System.AddressArbiter.ArbitrateUnlock(Memory, MutexAddress);
 
-            ThreadState.X0 = 0;
+            if (Result != 0)
+            {
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
+            }
+
+            ThreadState.X0 = (ulong)Result;
         }
 
         private void SvcWaitProcessWideKeyAtomic(AThreadState ThreadState)
@@ -102,7 +164,7 @@ namespace Ryujinx.HLE.HOS.Kernel
             long  MutexAddress   = (long)ThreadState.X0;
             long  CondVarAddress = (long)ThreadState.X1;
             int   ThreadHandle   =  (int)ThreadState.X2;
-            ulong Timeout        =       ThreadState.X3;
+            long  Timeout        = (long)ThreadState.X3;
 
             Device.Log.PrintDebug(LogClass.KernelSvc,
                 "MutexAddress = 0x"   + MutexAddress  .ToString("x16") + ", " +
@@ -128,84 +190,52 @@ namespace Ryujinx.HLE.HOS.Kernel
                 return;
             }
 
-            KThread Thread = Process.HandleTable.GetData<KThread>(ThreadHandle);
+            long Result = System.AddressArbiter.WaitProcessWideKeyAtomic(
+                Memory,
+                MutexAddress,
+                CondVarAddress,
+                ThreadHandle,
+                Timeout);
 
-            if (Thread == null)
+            if (Result != 0)
             {
-                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid thread handle 0x{ThreadHandle:x8}!");
-
-                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
-
-                return;
+                if (Result == MakeError(ErrorModule.Kernel, KernelErr.Timeout))
+                {
+                    Device.Log.PrintDebug(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
+                }
+                else
+                {
+                    Device.Log.PrintWarning(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
+                }
             }
 
-            KThread WaitThread = Process.GetThread(ThreadState.Tpidr);
-
-            if (!CondVarWait(WaitThread, ThreadHandle, MutexAddress, CondVarAddress, Timeout))
-            {
-                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.Timeout);
-
-                return;
-            }
-
-            ThreadState.X0 = 0;
+            ThreadState.X0 = (ulong)Result;
         }
 
         private void SvcSignalProcessWideKey(AThreadState ThreadState)
         {
-            long CondVarAddress = (long)ThreadState.X0;
+            long Address = (long)ThreadState.X0;
             int  Count   =  (int)ThreadState.X1;
 
             Device.Log.PrintDebug(LogClass.KernelSvc,
-                "CondVarAddress = 0x" + CondVarAddress.ToString("x16") + ", " +
+                "Address = 0x" + Address.ToString("x16") + ", " +
                 "Count = 0x"   + Count  .ToString("x8"));
 
-            KThread CurrThread = Process.GetThread(ThreadState.Tpidr);
-
-            CondVarSignal(ThreadState, CurrThread, CondVarAddress, Count);
+            System.AddressArbiter.SignalProcessWideKey(Process, Memory, Address, Count);
 
             ThreadState.X0 = 0;
         }
 
-        private void MutexLock(
-            KThread CurrThread,
-            KThread WaitThread,
-            int     OwnerThreadHandle,
-            int     WaitThreadHandle,
-            long    MutexAddress)
-        {
-            lock (Process.ThreadSyncLock)
-            {
-                int MutexValue = Memory.ReadInt32(MutexAddress);
-
-                Device.Log.PrintDebug(LogClass.KernelSvc, "MutexValue = 0x" + MutexValue.ToString("x8"));
-
-                if (MutexValue != (OwnerThreadHandle | MutexHasListenersMask))
-                {
-                    return;
-                }
-
-                CurrThread.WaitHandle   = WaitThreadHandle;
-                CurrThread.MutexAddress = MutexAddress;
-
-                InsertWaitingMutexThreadUnsafe(OwnerThreadHandle, WaitThread);
-            }
-
-            Device.Log.PrintDebug(LogClass.KernelSvc, "Entering wait state...");
-
-            Process.Scheduler.EnterWait(CurrThread);
-        }
-
         private void SvcWaitForAddress(AThreadState ThreadState)
         {
             long            Address =            (long)ThreadState.X0;
             ArbitrationType Type    = (ArbitrationType)ThreadState.X1;
             int             Value   =             (int)ThreadState.X2;
-            ulong           Timeout = ThreadState.X3;
+            long            Timeout =            (long)ThreadState.X3;
 
             Device.Log.PrintDebug(LogClass.KernelSvc,
                 "Address = 0x" + Address.ToString("x16") + ", " +
-                "ArbitrationType = 0x" + Type   .ToString()      + ", " +
+                "Type = "      + Type   .ToString()      + ", " +
                 "Value = 0x"   + Value  .ToString("x8")  + ", " +
                 "Timeout = 0x" + Timeout.ToString("x16"));
 
@@ -227,287 +257,93 @@ namespace Ryujinx.HLE.HOS.Kernel
                 return;
             }
 
+            long Result;
+
             switch (Type)
             {
                 case ArbitrationType.WaitIfLessThan:
-                    ThreadState.X0 = AddressArbiter.WaitForAddressIfLessThan(Process, ThreadState, Memory, Address, Value, Timeout, false);
+                    Result = System.AddressArbiter.WaitForAddressIfLessThan(Memory, Address, Value, false, Timeout);
                     break;
 
                 case ArbitrationType.DecrementAndWaitIfLessThan:
-                    ThreadState.X0 = AddressArbiter.WaitForAddressIfLessThan(Process, ThreadState, Memory, Address, Value, Timeout, true);
+                    Result = System.AddressArbiter.WaitForAddressIfLessThan(Memory, Address, Value, true, Timeout);
                     break;
 
                 case ArbitrationType.WaitIfEqual:
-                    ThreadState.X0 = AddressArbiter.WaitForAddressIfEqual(Process, ThreadState, Memory, Address, Value, Timeout);
+                    Result = System.AddressArbiter.WaitForAddressIfEqual(Memory, Address, Value, Timeout);
                     break;
 
                 default:
-                    ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidEnumValue);
-                    break;
-            }
-        }
-
-        private void MutexUnlock(KThread CurrThread, long MutexAddress)
-        {
-            lock (Process.ThreadSyncLock)
-            {
-                //This is the new thread that will now own the mutex.
-                //If no threads are waiting for the lock, then it should be null.
-                (KThread OwnerThread, int Count) = PopMutexThreadUnsafe(CurrThread, MutexAddress);
-
-                if (OwnerThread == CurrThread)
-                {
-                    throw new InvalidOperationException();
-                }
-
-                if (OwnerThread != null)
-                {
-                    //Remove all waiting mutex from the old owner,
-                    //and insert then on the new owner.
-                    UpdateMutexOwnerUnsafe(CurrThread, OwnerThread, MutexAddress);
-
-                    CurrThread.UpdatePriority();
-
-                    int HasListeners = Count >= 2 ? MutexHasListenersMask : 0;
-
-                    Memory.WriteInt32ToSharedAddr(MutexAddress, HasListeners | OwnerThread.WaitHandle);
-
-                    OwnerThread.WaitHandle     = 0;
-                    OwnerThread.MutexAddress   = 0;
-                    OwnerThread.CondVarAddress = 0;
-                    OwnerThread.MutexOwner     = null;
-
-                    OwnerThread.UpdatePriority();
-
-                    Process.Scheduler.WakeUp(OwnerThread);
-
-                    Device.Log.PrintDebug(LogClass.KernelSvc, "Gave mutex to thread id " + OwnerThread.ThreadId + "!");
-                }
-                else
-                {
-                    Memory.WriteInt32ToSharedAddr(MutexAddress, 0);
-
-                    Device.Log.PrintDebug(LogClass.KernelSvc, "No threads waiting mutex!");
-                }
-            }
-        }
-
-        private bool CondVarWait(
-            KThread WaitThread,
-            int     WaitThreadHandle,
-            long    MutexAddress,
-            long    CondVarAddress,
-            ulong   Timeout)
-        {
-            WaitThread.WaitHandle     = WaitThreadHandle;
-            WaitThread.MutexAddress   = MutexAddress;
-            WaitThread.CondVarAddress = CondVarAddress;
-
-            lock (Process.ThreadSyncLock)
-            {
-                MutexUnlock(WaitThread, MutexAddress);
-
-                WaitThread.CondVarSignaled = false;
-
-                Process.ThreadArbiterList.Add(WaitThread);
-            }
-
-            Device.Log.PrintDebug(LogClass.KernelSvc, "Entering wait state...");
-
-            if (Timeout != ulong.MaxValue)
-            {
-                Process.Scheduler.EnterWait(WaitThread, NsTimeConverter.GetTimeMs(Timeout));
-
-                lock (Process.ThreadSyncLock)
-                {
-                    if (!WaitThread.CondVarSignaled || WaitThread.MutexOwner != null)
-                    {
-                        if (WaitThread.MutexOwner != null)
-                        {
-                            WaitThread.MutexOwner.MutexWaiters.Remove(WaitThread);
-                            WaitThread.MutexOwner.UpdatePriority();
-
-                            WaitThread.MutexOwner = null;
-                        }
-
-                        Process.ThreadArbiterList.Remove(WaitThread);
-
-                        Device.Log.PrintDebug(LogClass.KernelSvc, "Timed out...");
-
-                        return false;
-                    }
-                }
-            }
-            else
-            {
-                Process.Scheduler.EnterWait(WaitThread);
-            }
-
-            return true;
-        }
-
-        private void CondVarSignal(
-            AThreadState ThreadState,
-            KThread      CurrThread,
-            long         CondVarAddress,
-            int          Count)
-        {
-            lock (Process.ThreadSyncLock)
-            {
-                while (Count == -1 || Count-- > 0)
-                {
-                    KThread WaitThread = PopCondVarThreadUnsafe(CondVarAddress);
-
-                    if (WaitThread == null)
-                    {
-                        Device.Log.PrintDebug(LogClass.KernelSvc, "No more threads to wake up!");
-
+                    Result = MakeError(ErrorModule.Kernel, KernelErr.InvalidEnumValue);
                     break;
             }
 
-                    WaitThread.CondVarSignaled = true;
-
-                    long MutexAddress = WaitThread.MutexAddress;
-
-                    Memory.SetExclusive(ThreadState, MutexAddress);
-
-                    int MutexValue = Memory.ReadInt32(MutexAddress);
-
-                    while (MutexValue != 0)
+            if (Result != 0)
             {
-                        if (Memory.TestExclusive(ThreadState, MutexAddress))
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
+            }
+
+            ThreadState.X0 = (ulong)Result;
+        }
+
+        private void SvcSignalToAddress(AThreadState ThreadState)
         {
-                            //Wait until the lock is released.
-                            InsertWaitingMutexThreadUnsafe(MutexValue & ~MutexHasListenersMask, WaitThread);
+            long       Address =       (long)ThreadState.X0;
+            SignalType Type    = (SignalType)ThreadState.X1;
+            int        Value   =        (int)ThreadState.X2;
+            int        Count   =        (int)ThreadState.X3;
 
-                            Memory.WriteInt32(MutexAddress, MutexValue | MutexHasListenersMask);
+            Device.Log.PrintDebug(LogClass.KernelSvc,
+                "Address = 0x" + Address.ToString("x16") + ", " +
+                "Type = "      + Type   .ToString()      + ", " +
+                "Value = 0x"   + Value  .ToString("x8")  + ", " +
+                "Count = 0x"   + Count  .ToString("x8"));
 
-                            Memory.ClearExclusiveForStore(ThreadState);
-
-                            break;
-                        }
-
-                        Memory.SetExclusive(ThreadState, MutexAddress);
-
-                        MutexValue = Memory.ReadInt32(MutexAddress);
-                    }
-
-                    Device.Log.PrintDebug(LogClass.KernelSvc, "MutexValue = 0x" + MutexValue.ToString("x8"));
-
-                    if (MutexValue == 0)
+            if (IsPointingInsideKernel(Address))
             {
-                        //Give the lock to this thread.
-                        Memory.WriteInt32ToSharedAddr(MutexAddress, WaitThread.WaitHandle);
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid address 0x{Address:x16}!");
 
-                        WaitThread.WaitHandle     = 0;
-                        WaitThread.MutexAddress   = 0;
-                        WaitThread.CondVarAddress = 0;
-
-                        WaitThread.MutexOwner?.UpdatePriority();
-
-                        WaitThread.MutexOwner = null;
-
-                        Process.Scheduler.WakeUp(WaitThread);
-                    }
-                }
-            }
-        }
-
-        private void UpdateMutexOwnerUnsafe(KThread CurrThread, KThread NewOwner, long MutexAddress)
-        {
-            //Go through all threads waiting for the mutex,
-            //and update the MutexOwner field to point to the new owner.
-            for (int Index = 0; Index < CurrThread.MutexWaiters.Count; Index++)
-            {
-                KThread Thread = CurrThread.MutexWaiters[Index];
-
-                if (Thread.MutexAddress == MutexAddress)
-                {
-                    CurrThread.MutexWaiters.RemoveAt(Index--);
-
-                    InsertWaitingMutexThreadUnsafe(NewOwner, Thread);
-                }
-            }
-        }
-
-        private void InsertWaitingMutexThreadUnsafe(int OwnerThreadHandle, KThread WaitThread)
-        {
-            KThread OwnerThread = Process.HandleTable.GetData<KThread>(OwnerThreadHandle);
-
-            if (OwnerThread == null)
-            {
-                Device.Log.PrintWarning(LogClass.KernelSvc, $"Invalid thread handle 0x{OwnerThreadHandle:x8}!");
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm);
 
                 return;
             }
 
-            InsertWaitingMutexThreadUnsafe(OwnerThread, WaitThread);
+            if (IsAddressNotWordAligned(Address))
+            {
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Unaligned address 0x{Address:x16}!");
+
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidAddress);
+
+                return;
             }
 
-        private void InsertWaitingMutexThreadUnsafe(KThread OwnerThread, KThread WaitThread)
-        {
-            WaitThread.MutexOwner = OwnerThread;
+            long Result;
 
-            if (!OwnerThread.MutexWaiters.Contains(WaitThread))
+            switch (Type)
             {
-                OwnerThread.MutexWaiters.Add(WaitThread);
+                case SignalType.Signal:
+                    Result = System.AddressArbiter.Signal(Address, Count);
+                    break;
 
-                OwnerThread.UpdatePriority();
-            }
+                case SignalType.SignalAndIncrementIfEqual:
+                    Result = System.AddressArbiter.SignalAndIncrementIfEqual(Memory, Address, Value, Count);
+                    break;
+
+                case SignalType.SignalAndModifyIfEqual:
+                    Result = System.AddressArbiter.SignalAndModifyIfEqual(Memory, Address, Value, Count);
+                    break;
+
+                default:
+                    Result = MakeError(ErrorModule.Kernel, KernelErr.InvalidEnumValue);
+                    break;
             }
 
-        private (KThread, int) PopMutexThreadUnsafe(KThread OwnerThread, long MutexAddress)
+            if (Result != 0)
             {
-            int Count = 0;
-
-            KThread WakeThread = null;
-
-            foreach (KThread Thread in OwnerThread.MutexWaiters)
-            {
-                if (Thread.MutexAddress != MutexAddress)
-                {
-                    continue;
+                Device.Log.PrintWarning(LogClass.KernelSvc, $"Operation failed with error 0x{Result:x}!");
             }
 
-                if (WakeThread == null || Thread.ActualPriority < WakeThread.ActualPriority)
-                {
-                    WakeThread = Thread;
-                }
-
-                Count++;
-            }
-
-            if (WakeThread != null)
-            {
-                OwnerThread.MutexWaiters.Remove(WakeThread);
-            }
-
-            return (WakeThread, Count);
-        }
-
-        private KThread PopCondVarThreadUnsafe(long CondVarAddress)
-        {
-            KThread WakeThread = null;
-
-            foreach (KThread Thread in Process.ThreadArbiterList)
-            {
-                if (Thread.CondVarAddress != CondVarAddress)
-                {
-                    continue;
-                }
-
-                if (WakeThread == null || Thread.ActualPriority < WakeThread.ActualPriority)
-                {
-                    WakeThread = Thread;
-                }
-            }
-
-            if (WakeThread != null)
-            {
-                Process.ThreadArbiterList.Remove(WakeThread);
-            }
-
-            return WakeThread;
+            ThreadState.X0 = (ulong)Result;
         }
 
         private bool IsPointingInsideKernel(long Address)

Ryujinx.HLE/HOS/Kernel/ThreadQueue.cs

@@ -1,158 +0,0 @@
-namespace Ryujinx.HLE.HOS.Kernel
-{
-    class ThreadQueue
-    {
-        private const int LowestPriority = 0x3f;
-
-        private SchedulerThread Head;
-
-        private object ListLock;
-
-        public ThreadQueue()
-        {
-            ListLock = new object();
-        }
-
-        public void Push(SchedulerThread Wait)
-        {
-            lock (ListLock)
-            {
-                //Ensure that we're not creating circular references
-                //by adding a thread that is already on the list.
-                if (HasThread(Wait))
-                {
-                    return;
-                }
-
-                if (Head == null || Head.Thread.ActualPriority >= Wait.Thread.ActualPriority)
-                {
-                    Wait.Next = Head;
-
-                    Head = Wait;
-
-                    return;
-                }
-
-                SchedulerThread Curr = Head;
-
-                while (Curr.Next != null)
-                {
-                    if (Curr.Next.Thread.ActualPriority >= Wait.Thread.ActualPriority)
-                    {
-                        break;
-                    }
-
-                    Curr = Curr.Next;
-                }
-
-                Wait.Next = Curr.Next;
-                Curr.Next = Wait;
-            }
-        }
-
-        public SchedulerThread Pop(int Core, int MinPriority = LowestPriority)
-        {
-            lock (ListLock)
-            {
-                int CoreMask = 1 << Core;
-
-                SchedulerThread Prev = null;
-                SchedulerThread Curr = Head;
-
-                while (Curr != null)
-                {
-                    KThread Thread = Curr.Thread;
-
-                    if (Thread.ActualPriority <= MinPriority && (Thread.CoreMask & CoreMask) != 0)
-                    {
-                        if (Prev != null)
-                        {
-                            Prev.Next = Curr.Next;
-                        }
-                        else
-                        {
-                            Head = Head.Next;
-                        }
-
-                        break;
-                    }
-
-                    Prev = Curr;
-                    Curr = Curr.Next;
-                }
-
-                return Curr;
-            }
-        }
-
-        public bool Remove(SchedulerThread Thread)
-        {
-            lock (ListLock)
-            {
-                if (Head == null)
-                {
-                    return false;
-                }
-                else if (Head == Thread)
-                {
-                    Head = Head.Next;
-
-                    return true;
-                }
-
-                SchedulerThread Prev = Head;
-                SchedulerThread Curr = Head.Next;
-
-                while (Curr != null)
-                {
-                    if (Curr == Thread)
-                    {
-                        Prev.Next = Curr.Next;
-
-                        return true;
-                    }
-
-                    Prev = Curr;
-                    Curr = Curr.Next;
-                }
-
-                return false;
-            }
-        }
-
-        public bool Resort(SchedulerThread Thread)
-        {
-            lock (ListLock)
-            {
-                if (Remove(Thread))
-                {
-                    Push(Thread);
-
-                    return true;
-                }
-
-                return false;
-            }
-        }
-
-        public bool HasThread(SchedulerThread Thread)
-        {
-            lock (ListLock)
-            {
-                SchedulerThread Curr = Head;
-
-                while (Curr != null)
-                {
-                    if (Curr == Thread)
-                    {
-                        return true;
-                    }
-
-                    Curr = Curr.Next;
-                }
-
-                return false;
-            }
-        }
-    }
-}

Ryujinx.HLE/HOS/Kernel/ThreadSchedState.cs

@@ -0,0 +1,15 @@
+namespace Ryujinx.HLE.HOS.Kernel
+{
+    enum ThreadSchedState : byte
+    {
+        LowNibbleMask   = 0xf,
+        HighNibbleMask  = 0xf0,
+        ExceptionalMask = 0x70,
+        ForcePauseFlag  = 0x20,
+
+        None               = 0,
+        Paused             = 1,
+        Running            = 2,
+        TerminationPending = 3
+    }
+}

Ryujinx.HLE/HOS/Process.cs

@@ -40,12 +40,6 @@ namespace Ryujinx.HLE.HOS
 
         private List<KTlsPageManager> TlsPages;
 
-        public KProcessScheduler Scheduler { get; private set; }
-
-        public List<KThread> ThreadArbiterList { get; private set; }
-
-        public object ThreadSyncLock { get; private set; }
-
         public Npdm MetaData { get; private set; }
 
         public KProcessHandleTable HandleTable { get; private set; }
@@ -62,14 +56,11 @@ namespace Ryujinx.HLE.HOS
 
         private long ImageBase;
 
-        private bool ShouldDispose;
-
         private bool Disposed;
 
-        public Process(Switch Device, KProcessScheduler Scheduler, int ProcessId, Npdm MetaData)
+        public Process(Switch Device, int ProcessId, Npdm MetaData)
         {
             this.Device    = Device;
-            this.Scheduler = Scheduler;
             this.MetaData  = MetaData;
             this.ProcessId = ProcessId;
 
@@ -79,13 +70,9 @@ namespace Ryujinx.HLE.HOS
 
             TlsPages = new List<KTlsPageManager>();
 
-            ThreadArbiterList = new List<KThread>();
-
-            ThreadSyncLock = new object();
-
             HandleTable = new KProcessHandleTable();
 
-            AppletState = new AppletStateMgr();
+            AppletState = new AppletStateMgr(Device.System);
 
             SvcHandler = new SvcHandler(Device, this);
 
@@ -171,15 +158,17 @@ namespace Ryujinx.HLE.HOS
 
                 Homebrew.WriteHbAbiData(Memory, HbAbiDataPosition, Handle, SwitchPath);
 
-                MainThread.Thread.ThreadState.X0 = (ulong)HbAbiDataPosition;
-                MainThread.Thread.ThreadState.X1 = ulong.MaxValue;
+                MainThread.Context.ThreadState.X0 = (ulong)HbAbiDataPosition;
+                MainThread.Context.ThreadState.X1 = ulong.MaxValue;
             }
 
-            Scheduler.StartThread(MainThread);
+            MainThread.TimeUp();
 
             return true;
         }
 
+        private int ThreadIdCtr = 1;
+
         public int MakeThread(
             long EntryPoint,
             long StackTop,
@@ -196,9 +185,9 @@ namespace Ryujinx.HLE.HOS
 
             long Tpidr = GetFreeTls();
 
-            int ThreadId = (int)((Tpidr - MemoryManager.TlsIoRegionStart) / 0x200) + 1;
+            int ThreadId = ThreadIdCtr++; //(int)((Tpidr - MemoryManager.TlsIoRegionStart) / 0x200) + 1;
 
-            KThread Thread = new KThread(CpuThread, this, ProcessorId, Priority, ThreadId);
+            KThread Thread = new KThread(CpuThread, this, Device.System, ProcessorId, Priority, ThreadId);
 
             Thread.LastPc = EntryPoint;
 
@@ -211,6 +200,7 @@ namespace Ryujinx.HLE.HOS
             CpuThread.ThreadState.X1  = (ulong)Handle;
             CpuThread.ThreadState.X31 = (ulong)StackTop;
 
+            CpuThread.ThreadState.Interrupt += InterruptHandler;
             CpuThread.ThreadState.Break     += BreakHandler;
             CpuThread.ThreadState.SvcCall   += SvcHandler.SvcCall;
             CpuThread.ThreadState.Undefined += UndefinedHandler;
@@ -248,6 +238,11 @@ namespace Ryujinx.HLE.HOS
             return Position;
         }
 
+        private void InterruptHandler(object sender, EventArgs e)
+        {
+            Device.System.Scheduler.ContextSwitch();
+        }
+
         private void BreakHandler(object sender, AInstExceptionEventArgs e)
         {
             throw new GuestBrokeExecutionException();
@@ -359,10 +354,6 @@ namespace Ryujinx.HLE.HOS
             if (sender is AThread Thread)
             {
                 Threads.TryRemove(Thread.ThreadState.Tpidr, out KThread KernelThread);
-
-                Scheduler.RemoveThread(KernelThread);
-
-                KernelThread.WaitEvent.Set();
             }
 
             if (Threads.Count == 0)
@@ -400,8 +391,6 @@ namespace Ryujinx.HLE.HOS
 
             INvDrvServices.UnloadProcess(this);
 
-            AppletState.Dispose();
-
             if (NeedsHbAbi && Executables.Count > 0 && Executables[0].FilePath.EndsWith(Homebrew.TemporaryNroSuffix))
             {
                 File.Delete(Executables[0].FilePath);
@@ -423,9 +412,7 @@ namespace Ryujinx.HLE.HOS
                 {
                     foreach (KThread Thread in Threads.Values)
                     {
-                        Thread.Thread.StopExecution();
-
-                        Scheduler.ForceWakeUp(Thread);
+                        Device.System.Scheduler.StopThread(Thread);
                     }
                 }
                 else

Ryujinx.HLE/HOS/Services/Am/IApplicationProxy.cs

@@ -26,14 +26,14 @@ namespace Ryujinx.HLE.HOS.Services.Am
 
         public long GetCommonStateGetter(ServiceCtx Context)
         {
-            MakeObject(Context, new ICommonStateGetter());
+            MakeObject(Context, new ICommonStateGetter(Context.Device.System));
 
             return 0;
         }
 
         public long GetSelfController(ServiceCtx Context)
         {
-            MakeObject(Context, new ISelfController());
+            MakeObject(Context, new ISelfController(Context.Device.System));
 
             return 0;
         }

Ryujinx.HLE/HOS/Services/Am/ICommonStateGetter.cs

@@ -15,7 +15,7 @@ namespace Ryujinx.HLE.HOS.Services.Am
 
         private KEvent DisplayResolutionChangeEvent;
 
-        public ICommonStateGetter()
+        public ICommonStateGetter(Horizon System)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -29,7 +29,7 @@ namespace Ryujinx.HLE.HOS.Services.Am
                 { 61, GetDefaultDisplayResolutionChangeEvent  }
             };
 
-            DisplayResolutionChangeEvent = new KEvent();
+            DisplayResolutionChangeEvent = new KEvent(System);
         }
 
         public long GetEventHandle(ServiceCtx Context)

Ryujinx.HLE/HOS/Services/Am/IHomeMenuFunctions.cs

@@ -13,7 +13,7 @@ namespace Ryujinx.HLE.HOS.Services.Am
 
         private KEvent ChannelEvent;
 
-        public IHomeMenuFunctions()
+        public IHomeMenuFunctions(Horizon System)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -22,7 +22,7 @@ namespace Ryujinx.HLE.HOS.Services.Am
             };
 
             //ToDo: Signal this Event somewhere in future.
-            ChannelEvent = new KEvent();
+            ChannelEvent = new KEvent(System);
         }
 
         public long RequestToGetForeground(ServiceCtx Context)

Ryujinx.HLE/HOS/Services/Am/ILibraryAppletAccessor.cs

@@ -13,7 +13,7 @@ namespace Ryujinx.HLE.HOS.Services.Am
 
         private KEvent StateChangedEvent;
 
-        public ILibraryAppletAccessor()
+        public ILibraryAppletAccessor(Horizon System)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -24,12 +24,12 @@ namespace Ryujinx.HLE.HOS.Services.Am
                 { 101, PopOutData                 }
             };
 
-            StateChangedEvent = new KEvent();
+            StateChangedEvent = new KEvent(System);
         }
 
         public long GetAppletStateChangedEvent(ServiceCtx Context)
         {
-            StateChangedEvent.WaitEvent.Set();
+            StateChangedEvent.Signal();
 
             int Handle = Context.Process.HandleTable.OpenHandle(StateChangedEvent);
 

Ryujinx.HLE/HOS/Services/Am/ILibraryAppletCreator.cs

@@ -20,7 +20,7 @@ namespace Ryujinx.HLE.HOS.Services.Am
 
         public long CreateLibraryApplet(ServiceCtx Context)
         {
-            MakeObject(Context, new ILibraryAppletAccessor());
+            MakeObject(Context, new ILibraryAppletAccessor(Context.Device.System));
 
             return 0;
         }

Ryujinx.HLE/HOS/Services/Am/ISelfController.cs

@@ -13,7 +13,7 @@ namespace Ryujinx.HLE.HOS.Services.Am
 
         private KEvent LaunchableEvent;
 
-        public ISelfController()
+        public ISelfController(Horizon System)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -31,7 +31,7 @@ namespace Ryujinx.HLE.HOS.Services.Am
                 { 50, SetHandlesRequestToDisplay            }
             };
 
-            LaunchableEvent = new KEvent();
+            LaunchableEvent = new KEvent(System);
         }
 
         public long Exit(ServiceCtx Context)
@@ -57,7 +57,7 @@ namespace Ryujinx.HLE.HOS.Services.Am
 
         public long GetLibraryAppletLaunchableEvent(ServiceCtx Context)
         {
-            LaunchableEvent.WaitEvent.Set();
+            LaunchableEvent.Signal();
 
             int Handle = Context.Process.HandleTable.OpenHandle(LaunchableEvent);
 

Ryujinx.HLE/HOS/Services/Am/ISystemAppletProxy.cs

@@ -28,14 +28,14 @@ namespace Ryujinx.HLE.HOS.Services.Am
 
         public long GetCommonStateGetter(ServiceCtx Context)
         {
-            MakeObject(Context, new ICommonStateGetter());
+            MakeObject(Context, new ICommonStateGetter(Context.Device.System));
 
             return 0;
         }
 
         public long GetSelfController(ServiceCtx Context)
         {
-            MakeObject(Context, new ISelfController());
+            MakeObject(Context, new ISelfController(Context.Device.System));
 
             return 0;
         }
@@ -70,7 +70,7 @@ namespace Ryujinx.HLE.HOS.Services.Am
 
         public long GetHomeMenuFunctions(ServiceCtx Context)
         {
-            MakeObject(Context, new IHomeMenuFunctions());
+            MakeObject(Context, new IHomeMenuFunctions(Context.Device.System));
 
             return 0;
         }

Ryujinx.HLE/HOS/Services/Aud/AudioOut/IAudioOut.cs

@@ -155,8 +155,6 @@ namespace Ryujinx.HLE.HOS.Services.Aud.AudioOut
             if (Disposing)
             {
                 AudioOut.CloseTrack(Track);
-
-                ReleaseEvent.Dispose();
             }
         }
     }

Ryujinx.HLE/HOS/Services/Aud/AudioRenderer/IAudioRenderer.cs

@@ -38,7 +38,11 @@ namespace Ryujinx.HLE.HOS.Services.Aud.AudioRenderer
 
         private int Track;
 
-        public IAudioRenderer(AMemory Memory, IAalOutput AudioOut, AudioRendererParameter Params)
+        public IAudioRenderer(
+            Horizon                System,
+            AMemory                Memory,
+            IAalOutput             AudioOut,
+            AudioRendererParameter Params)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -48,7 +52,7 @@ namespace Ryujinx.HLE.HOS.Services.Aud.AudioRenderer
                 { 7, QuerySystemEvent           }
             };
 
-            UpdateEvent = new KEvent();
+            UpdateEvent = new KEvent(System);
 
             this.Memory   = Memory;
             this.AudioOut = AudioOut;
@@ -68,7 +72,7 @@ namespace Ryujinx.HLE.HOS.Services.Aud.AudioRenderer
 
         private void AudioCallback()
         {
-            UpdateEvent.WaitEvent.Set();
+            UpdateEvent.Signal();
         }
 
         private static T[] CreateArray<T>(int Size) where T : new()
@@ -310,8 +314,6 @@ namespace Ryujinx.HLE.HOS.Services.Aud.AudioRenderer
             if (Disposing)
             {
                 AudioOut.CloseTrack(Track);
-
-                UpdateEvent.Dispose();
             }
         }
     }

Ryujinx.HLE/HOS/Services/Aud/IAudioDevice.cs

@@ -15,7 +15,7 @@ namespace Ryujinx.HLE.HOS.Services.Aud
 
         private KEvent SystemEvent;
 
-        public IAudioDevice()
+        public IAudioDevice(Horizon System)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -32,10 +32,10 @@ namespace Ryujinx.HLE.HOS.Services.Aud
                 { 12, QueryAudioDeviceOutputEvent    }
             };
 
-            SystemEvent = new KEvent();
+            SystemEvent = new KEvent(System);
 
             //TODO: We shouldn't be signaling this here.
-            SystemEvent.WaitEvent.Set();
+            SystemEvent.Signal();
         }
 
         public long ListAudioDeviceName(ServiceCtx Context)

Ryujinx.HLE/HOS/Services/Aud/IAudioOutManager.cs

@@ -146,11 +146,11 @@ namespace Ryujinx.HLE.HOS.Services.Aud
                 Channels = DefaultChannelsCount;
             }
 
-            KEvent ReleaseEvent = new KEvent();
+            KEvent ReleaseEvent = new KEvent(Context.Device.System);
 
             ReleaseCallback Callback = () =>
             {
-                ReleaseEvent.WaitEvent.Set();
+                ReleaseEvent.Signal();
             };
 
             IAalOutput AudioOut = Context.Device.AudioOut;

Ryujinx.HLE/HOS/Services/Aud/IAudioRendererManager.cs

@@ -40,7 +40,11 @@ namespace Ryujinx.HLE.HOS.Services.Aud
 
             AudioRendererParameter Params = GetAudioRendererParameter(Context);
 
-            MakeObject(Context, new IAudioRenderer(Context.Memory, AudioOut, Params));
+            MakeObject(Context, new IAudioRenderer(
+                Context.Device.System,
+                Context.Memory,
+                AudioOut,
+                Params));
 
             return 0;
         }
@@ -161,7 +165,7 @@ namespace Ryujinx.HLE.HOS.Services.Aud
         {
             long UserId = Context.RequestData.ReadInt64();
 
-            MakeObject(Context, new IAudioDevice());
+            MakeObject(Context, new IAudioDevice(Context.Device.System));
 
             return 0;
         }

Ryujinx.HLE/HOS/Services/Hid/IHidServer.cs

@@ -2,12 +2,11 @@ using Ryujinx.HLE.HOS.Ipc;
 using Ryujinx.HLE.HOS.Kernel;
 using Ryujinx.HLE.Input;
 using Ryujinx.HLE.Logging;
-using System;
 using System.Collections.Generic;
 
 namespace Ryujinx.HLE.HOS.Services.Hid
 {
-    class IHidServer : IpcService, IDisposable
+    class IHidServer : IpcService
     {
         private Dictionary<int, ServiceProcessRequest> m_Commands;
 
@@ -15,7 +14,7 @@ namespace Ryujinx.HLE.HOS.Services.Hid
 
         public override IReadOnlyDictionary<int, ServiceProcessRequest> Commands => m_Commands;
 
-        public IHidServer()
+        public IHidServer(Horizon System)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -45,7 +44,7 @@ namespace Ryujinx.HLE.HOS.Services.Hid
                 { 206, SendVibrationValues                     }
             };
 
-            NpadStyleSetUpdateEvent = new KEvent();
+            NpadStyleSetUpdateEvent = new KEvent(System);
         }
 
         public long CreateAppletResource(ServiceCtx Context)
@@ -282,18 +281,5 @@ namespace Ryujinx.HLE.HOS.Services.Hid
 
             return 0;
         }
-
-        public void Dispose()
-        {
-            Dispose(true);
-        }
-
-        protected virtual void Dispose(bool Disposing)
-        {
-            if (Disposing)
-            {
-                NpadStyleSetUpdateEvent.Dispose();
-            }
-        }
     }
 }

Ryujinx.HLE/HOS/Services/Nfp/IUser.cs

@@ -24,7 +24,7 @@ namespace Ryujinx.HLE.HOS.Services.Nfp
 
         private KEvent AvailabilityChangeEvent;
 
-        public IUser()
+        public IUser(Horizon System)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -37,9 +37,9 @@ namespace Ryujinx.HLE.HOS.Services.Nfp
                 { 23, AttachAvailabilityChangeEvent }
             };
 
-            ActivateEvent           = new KEvent();
-            DeactivateEvent         = new KEvent();
-            AvailabilityChangeEvent = new KEvent();
+            ActivateEvent           = new KEvent(System);
+            DeactivateEvent         = new KEvent(System);
+            AvailabilityChangeEvent = new KEvent(System);
         }
 
         public long Initialize(ServiceCtx Context)

Ryujinx.HLE/HOS/Services/Nfp/IUserManager.cs

@@ -19,7 +19,7 @@ namespace Ryujinx.HLE.HOS.Services.Nfp
 
         public long GetUserInterface(ServiceCtx Context)
         {
-            MakeObject(Context, new IUser());
+            MakeObject(Context, new IUser(Context.Device.System));
 
             return 0;
         }

Ryujinx.HLE/HOS/Services/Nifm/IGeneralService.cs

@@ -30,7 +30,7 @@ namespace Ryujinx.HLE.HOS.Services.Nifm
         {
             int Unknown = Context.RequestData.ReadInt32();
 
-            MakeObject(Context, new IRequest());
+            MakeObject(Context, new IRequest(Context.Device.System));
 
             Context.Device.Log.PrintStub(LogClass.ServiceNifm, "Stubbed.");
 

Ryujinx.HLE/HOS/Services/Nifm/IRequest.cs

@@ -1,12 +1,11 @@
 using Ryujinx.HLE.HOS.Ipc;
 using Ryujinx.HLE.HOS.Kernel;
 using Ryujinx.HLE.Logging;
-using System;
 using System.Collections.Generic;
 
 namespace Ryujinx.HLE.HOS.Services.Nifm
 {
-    class IRequest : IpcService, IDisposable
+    class IRequest : IpcService
     {
         private Dictionary<int, ServiceProcessRequest> m_Commands;
 
@@ -15,7 +14,7 @@ namespace Ryujinx.HLE.HOS.Services.Nifm
         private KEvent Event0;
         private KEvent Event1;
 
-        public IRequest()
+        public IRequest(Horizon System)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -27,8 +26,8 @@ namespace Ryujinx.HLE.HOS.Services.Nifm
                 { 11, SetConnectionConfirmationOption }
             };
 
-            Event0 = new KEvent();
-            Event1 = new KEvent();
+            Event0 = new KEvent(System);
+            Event1 = new KEvent(System);
         }
 
         public long GetRequestState(ServiceCtx Context)
@@ -77,19 +76,5 @@ namespace Ryujinx.HLE.HOS.Services.Nifm
 
             return 0;
         }
-
-        public void Dispose()
-        {
-            Dispose(true);
-        }
-
-        protected virtual void Dispose(bool Disposing)
-        {
-            if (Disposing)
-            {
-                Event0.Dispose();
-                Event1.Dispose();
-            }
-        }
     }
 }

Ryujinx.HLE/HOS/Services/Nv/INvDrvServices.cs

@@ -12,7 +12,7 @@ using System.Collections.Generic;
 
 namespace Ryujinx.HLE.HOS.Services.Nv
 {
-    class INvDrvServices : IpcService, IDisposable
+    class INvDrvServices : IpcService
     {
         private delegate int IoctlProcessor(ServiceCtx Context, int Cmd);
 
@@ -34,7 +34,7 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
         private KEvent Event;
 
-        public INvDrvServices()
+        public INvDrvServices(Horizon System)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -48,7 +48,7 @@ namespace Ryujinx.HLE.HOS.Services.Nv
                 { 13, FinishInitialize }
             };
 
-            Event = new KEvent();
+            Event = new KEvent(System);
         }
 
         static INvDrvServices()
@@ -214,18 +214,5 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
             NvMapIoctl.UnloadProcess(Process);
         }
-
-        public void Dispose()
-        {
-            Dispose(true);
-        }
-
-        protected virtual void Dispose(bool Disposing)
-        {
-            if (Disposing)
-            {
-                Event.Dispose();
-            }
-        }
     }
 }

Ryujinx.HLE/HOS/Services/ServiceFactory.cs

@@ -26,7 +26,7 @@ namespace Ryujinx.HLE.HOS.Services
 {
     static class ServiceFactory
     {
-        public static IpcService MakeService(string Name)
+        public static IpcService MakeService(Horizon System, string Name)
         {
             switch (Name)
             {
@@ -94,7 +94,7 @@ namespace Ryujinx.HLE.HOS.Services
                     return new IFileSystemProxy();
 
                 case "hid":
-                    return new IHidServer();
+                    return new IHidServer(System);
 
                 case "lm":
                     return new ILogService();
@@ -118,10 +118,10 @@ namespace Ryujinx.HLE.HOS.Services
                     return new IVulnerabilityManagerInterface();
 
                 case "nvdrv":
-                    return new INvDrvServices();
+                    return new INvDrvServices(System);
 
                 case "nvdrv:a":
-                    return new INvDrvServices();
+                    return new INvDrvServices(System);
 
                 case "pctl:s":
                     return new IParentalControlServiceFactory();

Ryujinx.HLE/HOS/Services/Sm/IUserInterface.cs

@@ -57,7 +57,7 @@ namespace Ryujinx.HLE.HOS.Services.Sm
                 return 0;
             }
 
-            KSession Session = new KSession(ServiceFactory.MakeService(Name), Name);
+            KSession Session = new KSession(ServiceFactory.MakeService(Context.Device.System, Name), Name);
 
             int Handle = Context.Process.HandleTable.OpenHandle(Session);
 

Ryujinx.HLE/HOS/Services/Vi/IApplicationDisplayService.cs

@@ -41,7 +41,9 @@ namespace Ryujinx.HLE.HOS.Services.Vi
 
         public long GetRelayService(ServiceCtx Context)
         {
-            MakeObject(Context, new IHOSBinderDriver(Context.Device.Gpu.Renderer));
+            MakeObject(Context, new IHOSBinderDriver(
+                Context.Device.System,
+                Context.Device.Gpu.Renderer));
 
             return 0;
         }
@@ -62,7 +64,9 @@ namespace Ryujinx.HLE.HOS.Services.Vi
 
         public long GetIndirectDisplayTransactionService(ServiceCtx Context)
         {
-            MakeObject(Context, new IHOSBinderDriver(Context.Device.Gpu.Renderer));
+            MakeObject(Context, new IHOSBinderDriver(
+                Context.Device.System,
+                Context.Device.Gpu.Renderer));
 
             return 0;
         }

Ryujinx.HLE/HOS/Services/Vi/IHOSBinderDriver.cs

@@ -17,7 +17,7 @@ namespace Ryujinx.HLE.HOS.Services.Vi
 
         private NvFlinger Flinger;
 
-        public IHOSBinderDriver(IGalRenderer Renderer)
+        public IHOSBinderDriver(Horizon System, IGalRenderer Renderer)
         {
             m_Commands = new Dictionary<int, ServiceProcessRequest>()
             {
@@ -27,9 +27,9 @@ namespace Ryujinx.HLE.HOS.Services.Vi
                 { 3, TransactParcelAuto }
             };
 
-            BinderEvent = new KEvent();
+            BinderEvent = new KEvent(System);
 
-            BinderEvent.WaitEvent.Set();
+            BinderEvent.Signal();
 
             Flinger = new NvFlinger(Renderer, BinderEvent);
         }
@@ -93,8 +93,6 @@ namespace Ryujinx.HLE.HOS.Services.Vi
         {
             if (Disposing)
             {
-                BinderEvent.Dispose();
-
                 Flinger.Dispose();
             }
         }

Ryujinx.HLE/HOS/Services/Vi/NvFlinger.cs

@@ -64,7 +64,7 @@ namespace Ryujinx.HLE.HOS.Services.Android
 
         private BufferEntry[] BufferQueue;
 
-        private ManualResetEvent WaitBufferFree;
+        private AutoResetEvent WaitBufferFree;
 
         private bool Disposed;
 
@@ -88,7 +88,7 @@ namespace Ryujinx.HLE.HOS.Services.Android
 
             BufferQueue = new BufferEntry[0x40];
 
-            WaitBufferFree = new ManualResetEvent(false);
+            WaitBufferFree = new AutoResetEvent(false);
         }
 
         public long ProcessParcelRequest(ServiceCtx Context, byte[] ParcelData, int Code)
@@ -220,6 +220,8 @@ namespace Ryujinx.HLE.HOS.Services.Android
 
             BufferQueue[Slot].State = BufferState.Free;
 
+            WaitBufferFree.Set();
+
             return MakeReplyParcel(Context, 0);
         }
 
@@ -336,21 +338,16 @@ namespace Ryujinx.HLE.HOS.Services.Android
         {
             BufferQueue[Slot].State = BufferState.Free;
 
-            BinderEvent.WaitEvent.Set();
+            BinderEvent.Signal();
 
-            lock (WaitBufferFree)
-            {
             WaitBufferFree.Set();
         }
-        }
 
         private int GetFreeSlotBlocking(int Width, int Height)
         {
             int Slot;
 
             do
-            {
-                lock (WaitBufferFree)
             {
                 if ((Slot = GetFreeSlot(Width, Height)) != -1)
                 {
@@ -362,9 +359,6 @@ namespace Ryujinx.HLE.HOS.Services.Android
                     break;
                 }
 
-                    WaitBufferFree.Reset();
-                }
-
                 WaitBufferFree.WaitOne();
             }
             while (!Disposed);
@@ -409,11 +403,7 @@ namespace Ryujinx.HLE.HOS.Services.Android
             {
                 Disposed = true;
 
-                lock (WaitBufferFree)
-                {
                 WaitBufferFree.Set();
-                }
-
                 WaitBufferFree.Dispose();
             }
         }

Ryujinx.HLE/HOS/SystemState/AppletStateMgr.cs

@@ -1,11 +1,10 @@
 using Ryujinx.HLE.HOS.Kernel;
 using Ryujinx.HLE.HOS.Services.Am;
-using System;
 using System.Collections.Concurrent;
 
 namespace Ryujinx.HLE.HOS.SystemState
 {
-    class AppletStateMgr : IDisposable
+    class AppletStateMgr
     {
         private ConcurrentQueue<MessageInfo> Messages;
 
@@ -13,11 +12,11 @@ namespace Ryujinx.HLE.HOS.SystemState
 
         public KEvent MessageEvent { get; private set; }
 
-        public AppletStateMgr()
+        public AppletStateMgr(Horizon System)
         {
             Messages = new ConcurrentQueue<MessageInfo>();
 
-            MessageEvent = new KEvent();
+            MessageEvent = new KEvent(System);
         }
 
         public void SetFocus(bool IsFocused)
@@ -33,30 +32,17 @@ namespace Ryujinx.HLE.HOS.SystemState
         {
             Messages.Enqueue(Message);
 
-            MessageEvent.WaitEvent.Set();
+            MessageEvent.Signal();
         }
 
         public bool TryDequeueMessage(out MessageInfo Message)
         {
             if (Messages.Count < 2)
             {
-                MessageEvent.WaitEvent.Reset();
+                MessageEvent.Reset();
             }
 
             return Messages.TryDequeue(out Message);
         }
-
-        public void Dispose()
-        {
-            Dispose(true);
-        }
-
-        protected virtual void Dispose(bool Disposing)
-        {
-            if (Disposing)
-            {
-                MessageEvent.Dispose();
-            }
-        }
     }
 }

Ryujinx/Config.cs

@@ -31,10 +31,6 @@ namespace Ryujinx
             Device.Log.SetEnable(LogLevel.Warning, Convert.ToBoolean(Parser.Value("Logging_Enable_Warn")));
             Device.Log.SetEnable(LogLevel.Error,   Convert.ToBoolean(Parser.Value("Logging_Enable_Error")));
 
-            Device.System.State.DockedMode = Convert.ToBoolean(Parser.Value("Docked_Mode"));
-
-            Device.EnableDeviceVsync = Convert.ToBoolean(Parser.Value("Enable_Vsync"));
-
             string[] FilteredLogClasses = Parser.Value("Logging_Filtered_Classes").Split(',', StringSplitOptions.RemoveEmptyEntries);
 
             //When the classes are specified on the list, we only
@@ -63,6 +59,15 @@ namespace Ryujinx
                 }
             }
 
+            Device.System.State.DockedMode = Convert.ToBoolean(Parser.Value("Docked_Mode"));
+
+            Device.EnableDeviceVsync = Convert.ToBoolean(Parser.Value("Enable_Vsync"));
+
+            if (Convert.ToBoolean(Parser.Value("Enable_MultiCore_Scheduling")))
+            {
+                Device.System.EnableMultiCoreScheduling();
+            }
+
             JoyConKeyboard = new JoyConKeyboard(
 
                 new JoyConKeyboardLeft

Ryujinx/Ryujinx.conf

@@ -28,6 +28,9 @@ Docked_Mode = false
 #Enable Game Vsync
 Enable_Vsync = true
 
+#Enable or Disable Multi-core scheduling of threads
+Enable_MultiCore_Scheduling = false
+
 #Controller Device Index
 GamePad_Index = 0