Make HLE disposable safely (#850)

* Make HLE disposable safely

This fix the oldest issue with the HLE code: the kernel side
disposability.

Changelog:

- Implement KProcess::UnpauseAndTerminateAllThreadsExcept, KThread::Terminate, KThread::TerminateCurrentProcess, KThread::PrepareForTermiation and the svc post handler accurately.
- Implement svcTerminateProcess and svcExitProcess. (both untested)
- Fix KHandleTable::Destroy not decrementing refcount of all objects stored in the table.
- Spawn a custom KProcess with the maximum priority to terminate every guest KProcess. (terminating kernel emulation safely)
- General system stability improvements to enhance the user's experience.

* Fix a typo in a comment in KProcess.cs

* Address gdk's comments

Ryujinx.HLE/HOS/Horizon.cs

@@ -754,21 +754,42 @@ namespace Ryujinx.HLE.HOS
         {
             if (disposing)
             {
-                // Force all threads to exit.
+                KProcess terminationProcess = new KProcess(this);
-                lock (Processes)
+
+                KThread terminationThread = new KThread(this);
+
+                terminationThread.Initialize(0, 0, 0, 3, 0, terminationProcess, ThreadType.Kernel, () =>
                 {
-                    foreach (KProcess process in Processes.Values)
+                    // Force all threads to exit.
+                    lock (Processes)
                     {
-                        process.StopAllThreads();
+                        foreach (KProcess process in Processes.Values)
+                        {
+                            process.Terminate();
+
+                            // Exit ourself now!
+                            Scheduler.ExitThread(terminationThread);
+                            Scheduler.GetCurrentThread().Exit();
+                            Scheduler.RemoveThread(terminationThread);
+                        }
                     }
+                });
+
+                terminationThread.Start();
+
+                // Signal the vsync event to avoid issues of KThread waiting on it.
+                if (Device.EnableDeviceVsync)
+                {
+                    Device.VsyncEvent.Set();
                 }
 
+                // This is needed as the IPC Dummy KThread is also counted in the ThreadCounter.
+                ThreadCounter.Signal();
+
                 // It's only safe to release resources once all threads
                 // have exited.
                 ThreadCounter.Signal();
-                //ThreadCounter.Wait(); // FIXME: Uncomment this
+                ThreadCounter.Wait();
-                // BODY: Right now, guest processes don't exit properly because the logic waits for them to exit.
-                // BODY: However, this doesn't happen when you close the main window so we need to find a way to make them exit gracefully
 
                 Scheduler.Dispose();
 

Ryujinx.HLE/HOS/Kernel/Process/KHandleTable.cs

@@ -272,6 +272,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
                             disposableObj.Dispose();
                         }
 
+                        entry.Obj.DecrementReferenceCount();
                         entry.Obj  = null;
                         entry.Next = _nextFreeEntry;
 

Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs

@@ -38,7 +38,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
 
         public ulong PersonalMmHeapPagesCount { get; private set; }
 
-        private ProcessState _state;
+        public ProcessState State { get; private set; }
 
         private object _processLock;
         private object _threadingLock;
@@ -383,7 +383,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
 
             Name = creationInfo.Name;
 
-            _state = ProcessState.Created;
+            State = ProcessState.Created;
 
             _creationTimestamp = PerformanceCounter.ElapsedMilliseconds;
 
@@ -579,7 +579,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
         {
             lock (_processLock)
             {
-                if (_state > ProcessState.CreatedAttached)
+                if (State > ProcessState.CreatedAttached)
                 {
                     return KernelResult.InvalidState;
                 }
@@ -733,8 +733,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
 
                 mainThread.SetEntryArguments(0, mainThreadHandle);
 
-                ProcessState oldState = _state;
+                ProcessState oldState = State;
-                ProcessState newState = _state != ProcessState.Created
+                ProcessState newState = State != ProcessState.Created
                     ? ProcessState.Attached
                     : ProcessState.Started;
 
@@ -768,9 +768,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
 
         private void SetState(ProcessState newState)
         {
-            if (_state != newState)
+            if (State != newState)
             {
-                _state    = newState;
+                State     = newState;
                 _signaled = true;
 
                 Signal();
@@ -820,6 +820,20 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
             }
         }
 
+        public void DecrementToZeroWhileTerminatingCurrent()
+        {
+            System.ThreadCounter.Signal();
+
+            while (Interlocked.Decrement(ref _threadCount) != 0)
+            {
+                Destroy();
+                TerminateCurrentProcess();
+            }
+
+            // Nintendo panic here because if it reaches this point, the current thread should be already dead.
+            // As we handle the death of the thread in the post SVC handler and inside the CPU emulator, we don't panic here.
+        }
+
         public ulong GetMemoryCapacity()
         {
             ulong totalCapacity = (ulong)ResourceLimit.GetRemainingValue(LimitableResource.Memory);
@@ -909,12 +923,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
 
             lock (_processLock)
             {
-                if (_state >= ProcessState.Started)
+                if (State >= ProcessState.Started)
                 {
-                    if (_state == ProcessState.Started  ||
+                    if (State == ProcessState.Started  ||
-                        _state == ProcessState.Crashed  ||
+                        State == ProcessState.Crashed  ||
-                        _state == ProcessState.Attached ||
+                        State == ProcessState.Attached ||
-                        _state == ProcessState.DebugSuspended)
+                        State == ProcessState.DebugSuspended)
                     {
                         SetState(ProcessState.Exiting);
 
@@ -933,23 +947,98 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
 
             if (shallTerminate)
             {
-                // UnpauseAndTerminateAllThreadsExcept(System.Scheduler.GetCurrentThread());
+                UnpauseAndTerminateAllThreadsExcept(System.Scheduler.GetCurrentThread());
 
                 HandleTable.Destroy();
 
-                SignalExitForDebugEvent();
+                SignalExitToDebugTerminated();
                 SignalExit();
             }
 
             return result;
         }
 
-        private void UnpauseAndTerminateAllThreadsExcept(KThread thread)
+        public void TerminateCurrentProcess()
         {
-            // TODO.
+            bool shallTerminate = false;
+
+            System.CriticalSection.Enter();
+
+            lock (_processLock)
+            {
+                if (State >= ProcessState.Started)
+                {
+                    if (State == ProcessState.Started ||
+                        State == ProcessState.Attached ||
+                        State == ProcessState.DebugSuspended)
+                    {
+                        SetState(ProcessState.Exiting);
+
+                        shallTerminate = true;
+                    }
+                }
+            }
+
+            System.CriticalSection.Leave();
+
+            if (shallTerminate)
+            {
+                UnpauseAndTerminateAllThreadsExcept(System.Scheduler.GetCurrentThread());
+
+                HandleTable.Destroy();
+
+                // NOTE: this is supposed to be called in receiving of the mailbox.
+                SignalExitToDebugExited();
+                SignalExit();
+            }
         }
 
-        private void SignalExitForDebugEvent()
+        private void UnpauseAndTerminateAllThreadsExcept(KThread currentThread)
+        {
+            lock (_threadingLock)
+            {
+                System.CriticalSection.Enter();
+
+                foreach (KThread thread in _threads)
+                {
+                    if ((thread.SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.TerminationPending)
+                    {
+                        thread.PrepareForTermination();
+                    }
+                }
+
+                System.CriticalSection.Leave();
+            }
+
+            KThread blockedThread = null;
+
+            lock (_threadingLock)
+            {
+                foreach (KThread thread in _threads)
+                {
+                    if (thread != currentThread && (thread.SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.TerminationPending)
+                    {
+                        thread.IncrementReferenceCount();
+
+                        blockedThread = thread;
+                        break;
+                    }
+                }
+            }
+
+            if (blockedThread != null)
+            {
+                blockedThread.Terminate();
+                blockedThread.DecrementReferenceCount();
+            }
+        }
+
+        private void SignalExitToDebugTerminated()
+        {
+            // TODO: Debug events.
+        }
+
+        private void SignalExitToDebugExited()
         {
             // TODO: Debug events.
         }
@@ -976,7 +1065,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
 
             lock (_processLock)
             {
-                if (_state != ProcessState.Exited && _signaled)
+                if (State != ProcessState.Exited && _signaled)
                 {
                     _signaled = false;
 
@@ -999,7 +1088,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
             {
                 foreach (KThread thread in _threads)
                 {
-                    thread.Context.Running = false;
+                    System.Scheduler.ExitThread(thread);
 
                     System.Scheduler.CoreManager.Set(thread.HostThread);
                 }

Ryujinx.HLE/HOS/Kernel/SupervisorCall/SvcHandler.cs

@@ -1,5 +1,6 @@
 using ARMeilleure.State;
 using Ryujinx.HLE.HOS.Kernel.Process;
+using Ryujinx.HLE.HOS.Kernel.Threading;
 using System;
 
 namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
@@ -29,6 +30,15 @@ namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
             ExecutionContext context = (ExecutionContext)sender;
 
             svcFunc(this, context);
+
+            PostSvcHandler();
+        }
+
+        private void PostSvcHandler()
+        {
+            KThread currentThread = _system.Scheduler.GetCurrentThread();
+
+            currentThread.HandlePostSyscall();
         }
     }
 }

Ryujinx.HLE/HOS/Kernel/SupervisorCall/SvcSystem.cs

@@ -17,9 +17,41 @@ namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
             ExitProcess();
         }
 
+        public KernelResult TerminateProcess64(int handle)
+        {
+            return TerminateProcess(handle);
+        }
+
+        private KernelResult TerminateProcess(int handle)
+        {
+            KProcess process = _process.HandleTable.GetObject<KProcess>(handle);
+
+            KernelResult result;
+
+            if (process != null)
+            {
+                if (process == _system.Scheduler.GetCurrentProcess())
+                {
+                    result = KernelResult.Success;
+                    process.DecrementToZeroWhileTerminatingCurrent();
+                }
+                else
+                {
+                    result = process.Terminate();
+                    process.DecrementReferenceCount();
+                }
+            }
+            else
+            {
+                result = KernelResult.InvalidHandle;
+            }
+
+            return result;
+        }
+
         private void ExitProcess()
         {
-            _system.Scheduler.GetCurrentProcess().Terminate();
+            _system.Scheduler.GetCurrentProcess().TerminateCurrentProcess();
         }
 
         public KernelResult SignalEvent64(int handle)
@@ -184,6 +216,15 @@ namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
             {
                 currentThread.PrintGuestStackTrace();
 
+                // As the process is exiting, this is probably caused by emulation termination.
+                if (currentThread.Owner.State == ProcessState.Exiting)
+                {
+                    return;
+                }
+
+                // TODO: Debug events.
+                currentThread.Owner.TerminateCurrentProcess();
+
                 throw new GuestBrokeExecutionException();
             }
             else

Ryujinx.HLE/HOS/Kernel/SupervisorCall/SvcTable.cs

@@ -74,7 +74,8 @@ namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
                 { 0x72, nameof(SvcHandler.ConnectToPort64)                 },
                 { 0x73, nameof(SvcHandler.SetProcessMemoryPermission64)    },
                 { 0x77, nameof(SvcHandler.MapProcessCodeMemory64)          },
-                { 0x78, nameof(SvcHandler.UnmapProcessCodeMemory64)        }
+                { 0x78, nameof(SvcHandler.UnmapProcessCodeMemory64)        },
+                { 0x7B, nameof(SvcHandler.TerminateProcess64)              }
             };
 
             _svcTable64 = new Action<SvcHandler, ExecutionContext>[0x80];

Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs

@@ -70,7 +70,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
         public ThreadSchedState SchedFlags { get; private set; }
 
-        public bool ShallBeTerminated { get; private set; }
+        private int _shallBeTerminated;
+
+        public bool ShallBeTerminated { get => _shallBeTerminated != 0; set => _shallBeTerminated = value ? 1 : 0; }
 
         public bool SyncCancelled { get; set; }
         public bool WaitingSync   { get; set; }
@@ -104,7 +106,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
             int        priority,
             int        defaultCpuCore,
             KProcess   owner,
-            ThreadType type = ThreadType.User)
+            ThreadType type = ThreadType.User,
+            ThreadStart customHostThreadStart = null)
         {
             if ((uint)type > 3)
             {
@@ -156,7 +159,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
                 is64Bits = true;
             }
 
-            HostThread = new Thread(() => ThreadStart(entrypoint));
+            HostThread = new Thread(customHostThreadStart == null ? () => ThreadStart(entrypoint) : customHostThreadStart);
 
             Context = new ARMeilleure.State.ExecutionContext();
 
@@ -182,6 +185,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
             ThreadUid = System.GetThreadUid();
 
+            HostThread.Name = $"Host Thread (thread id {ThreadUid})";
+
             _hasBeenInitialized = true;
 
             if (owner != null)
@@ -300,6 +305,100 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
             DecrementReferenceCount();
         }
 
+        public ThreadSchedState PrepareForTermination()
+        {
+            System.CriticalSection.Enter();
+
+            ThreadSchedState result;
+
+            if (Interlocked.CompareExchange(ref _shallBeTerminated, 1, 0) == 0)
+            {
+                if ((SchedFlags & ThreadSchedState.LowMask) == ThreadSchedState.None)
+                {
+                    SchedFlags = ThreadSchedState.TerminationPending;
+                }
+                else
+                {
+                    if (_forcePauseFlags != ThreadSchedState.None)
+                    {
+                        _forcePauseFlags &= ~ThreadSchedState.ThreadPauseFlag;
+
+                        ThreadSchedState oldSchedFlags = SchedFlags;
+
+                        SchedFlags &= ThreadSchedState.LowMask;
+
+                        AdjustScheduling(oldSchedFlags);
+                    }
+
+                    if (BasePriority >= 0x10)
+                    {
+                        SetPriority(0xF);
+                    }
+
+                    if ((SchedFlags & ThreadSchedState.LowMask) == ThreadSchedState.Running)
+                    {
+                        // TODO: GIC distributor stuffs (sgir changes ect)
+                    }
+
+                    SignaledObj   = null;
+                    ObjSyncResult = KernelResult.ThreadTerminating;
+
+                    ReleaseAndResume();
+                }
+            }
+
+            result = SchedFlags;
+
+            System.CriticalSection.Leave();
+
+            return result & ThreadSchedState.LowMask;
+        }
+
+        public void Terminate()
+        {
+            ThreadSchedState state = PrepareForTermination();
+
+            if (state != ThreadSchedState.TerminationPending)
+            {
+                System.Synchronization.WaitFor(new KSynchronizationObject[] { this }, -1, out _);
+            }
+        }
+
+        public void HandlePostSyscall()
+        {
+            ThreadSchedState state;
+
+            do
+            {
+                if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
+                {
+                    System.Scheduler.ExitThread(this);
+                    Exit();
+
+                    // As the death of the thread is handled by the CPU emulator, we differ from the official kernel and return here.
+                    break;
+                }
+
+                System.CriticalSection.Enter();
+
+                if (ShallBeTerminated || SchedFlags == ThreadSchedState.TerminationPending)
+                {
+                    state = ThreadSchedState.TerminationPending;
+                }
+                else
+                {
+                    if (_forcePauseFlags != ThreadSchedState.None)
+                    {
+                        CombineForcePauseFlags();
+                    }
+
+                    state = ThreadSchedState.Running;
+                }
+
+                System.CriticalSection.Leave();
+            } while (state == ThreadSchedState.TerminationPending);
+        }
+
         private void ExitImpl()
         {
             System.CriticalSection.Enter();