HLE/GeckoCode: Add new HLE hook exit trampoline

Dolphin emulates GeckoCodes by fiddling with the CPU state when a
VI Interrupt occurs. The problem with this is that we don't know
where the PC is so it's non-deterministic and not necessarily
suitable for use with the codehandler.

There are two options: Patch the game like Gecko OS either directly
or using HLE::Patch, or use a trampoline so we can branch from any
PC even if it would otherwise not be valid. The problem with Gecko OS
patches is there are 10 of them and they have to be configured
manually (i.e. Game INIs to would need to have a [Core]GeckoHookType
property).

HLE_Misc::GeckoReturnTrampoline enables the Code Handler to be
entered from anywhere, the trampoline restores all the registers that
had to be secretly saved to the stack.

Source/Core/Core/Boot/Boot.cpp

@@ -483,5 +483,9 @@ bool CBoot::BootUp()
   // Not part of the binary itself, but either we or Gecko OS might insert
   // this, and it doesn't clear the icache properly.
   HLE::Patch(Gecko::ENTRY_POINT, "GeckoCodehandler");
+  if (SConfig::GetInstance().bEnableCheats)
+  {
+    HLE::Patch(Gecko::HLE_TRAMPOLINE_ADDRESS, "GeckoHandlerReturnTrampoline");
+  }
   return true;
 }

Source/Core/Core/GeckoCode.cpp

@@ -18,7 +18,6 @@
 
 namespace Gecko
 {
-static constexpr u32 INSTALLER_END_ADDRESS = 0x80003000;
 static constexpr u32 CODE_SIZE = 8;
 
 // return true if a code exists
@@ -107,7 +106,7 @@ static Installation InstallCodeHandlerLocked()
   const u32 codelist_end_address = INSTALLER_END_ADDRESS;
 
   // Write a magic value to 'gameid' (codehandleronly does not actually read this).
-  // This value will be read back and modified over time by HLE_Misc::HLEGeckoCodehandler.
+  // This value will be read back and modified over time by HLE_Misc::GeckoCodeHandlerICacheFlush.
   PowerPC::HostWrite_U32(MAGIC_GAMEID, INSTALLER_BASE_ADDRESS);
 
   // Create GCT in memory
@@ -158,11 +157,25 @@ static Installation InstallCodeHandlerLocked()
   return Installation::Installed;
 }
 
-void RunCodeHandler()
+void RunCodeHandler(u32 msr_reg)
 {
   if (!SConfig::GetInstance().bEnableCheats)
     return;
 
+  // Dolphin's hook mechanism is less 'precise' than Gecko OS' which detects particular
+  // instruction sequences (uses configuration, not automatic) that only run once per frame
+  // which includes a BLR as one of the instructions. It then overwrites the BLR with a
+  // "B 0x800018A8" to establish the hook. Dolphin uses its own internal VI interrupt which
+  // means the PC is non-deterministic and could be anywhere.
+  UReg_MSR msr = msr_reg;
+  if (!msr.DR || !msr.IR)
+  {
+    WARN_LOG(ACTIONREPLAY, "GeckoCode: Skipping frame update. MSR.IR/DR is currently disabled. "
+                           "PC = 0x%08X, MSR = 0x%08X",
+             PC, msr_reg);
+    return;
+  }
+
   std::lock_guard<std::mutex> codes_lock(s_active_codes_lock);
   // Don't spam retry if the install failed. The corrupt / missing disk file is not likely to be
   // fixed within 1 frame of the last error.
@@ -180,11 +193,36 @@ void RunCodeHandler()
 
   // If the last block that just executed ended with a BLR instruction then we can intercept it and
   // redirect control into the Gecko Code Handler. The Code Handler will automatically BLR back to
-  // the original return address (which will still be in the link register) at the end.
-  if (PC == LR)
+  // the original return address (which will still be in the link register).
+  if (PC != LR)
   {
-    PC = NPC = ENTRY_POINT;
+    // We're at a random address in the middle of something so we have to do this the hard way.
+    // The codehandler will STMW all of the GPR registers, but we need to fix the Stack's Red
+    // Zone, the LR, PC (return address) and the volatile floating point registers.
+    // Build a function call stack frame.
+    u32 SFP = GPR(1);                     // Stack Frame Pointer
+    GPR(1) -= 224;                        // Stack's Red Zone
+    GPR(1) -= 16 + 2 * 14 * sizeof(u64);  // Our stack frame (HLE_Misc::GeckoReturnTrampoline)
+    GPR(1) -= 8;                          // Fake stack frame for codehandler
+    GPR(1) &= 0xFFFFFFF0;                 // Align stack to 16bytes
+    u32 SP = GPR(1);                      // Stack Pointer
+    PowerPC::HostWrite_U32(SP + 8, SP);
+    // SP + 4 is reserved for the codehandler to save LR to the stack.
+    PowerPC::HostWrite_U32(SFP, SP + 8);  // Real stack frame
+    PowerPC::HostWrite_U32(PC, SP + 12);
+    PowerPC::HostWrite_U32(LR, SP + 16);
+    // Registers FPR0->13 are volatile
+    for (int i = 0; i < 14; ++i)
+    {
+      PowerPC::HostWrite_U64(riPS0(i), SP + 24 + 2 * i * sizeof(u64));
+      PowerPC::HostWrite_U64(riPS1(i), SP + 24 + (2 * i + 1) * sizeof(u64));
+    }
+    LR = HLE_TRAMPOLINE_ADDRESS;
+    DEBUG_LOG(ACTIONREPLAY, "GeckoCodes: Initiating phantom branch-and-link. "
+                            "PC = 0x%08X, SP = 0x%08X, SFP = 0x%08X",
+              PC, SP, SFP);
   }
+  PC = NPC = ENTRY_POINT;
 }
 
 }  // namespace Gecko

Source/Core/Core/GeckoCode.h

@@ -35,19 +35,23 @@ public:
 
 // Installation address for codehandler.bin in the Game's RAM
 constexpr u32 INSTALLER_BASE_ADDRESS = 0x80001800;
-constexpr u32 ENTRY_POINT = 0x800018A8;
+constexpr u32 INSTALLER_END_ADDRESS = 0x80003000;
+constexpr u32 ENTRY_POINT = INSTALLER_BASE_ADDRESS + 0xA8;
+// If the GCT is max-length then this is the second word of the End code (0xF0000000 0x00000000)
+// If the table is shorter than the max-length then this address is unused / contains trash.
+constexpr u32 HLE_TRAMPOLINE_ADDRESS = INSTALLER_END_ADDRESS - 4;
 
-// This forms part of a communication protocol with HLE_Misc::HLEGeckoCodehandler.
+// This forms part of a communication protocol with HLE_Misc::GeckoCodeHandlerICacheFlush.
 // Basically, codehandleronly.s doesn't use ICBI like it's supposed to when patching the
 // game's code. This results in the JIT happily ignoring all code patches for blocks that
 // are already compiled. The hack for getting around that is that the first 5 frames after
 // the handler is installed (0xD01F1BAD -> +5 -> 0xD01F1BB2) cause full ICache resets.
 //
-// HLEGeckoCodehandler will increment this value 5 times then cease flushing the ICache to
+// GeckoCodeHandlerICacheFlush will increment this value 5 times then cease flushing the ICache to
 // preserve the emulation performance.
 constexpr u32 MAGIC_GAMEID = 0xD01F1BAD;
 
 void SetActiveCodes(const std::vector<GeckoCode>& gcodes);
-void RunCodeHandler();
+void RunCodeHandler(u32 msr_reg);
 
 }  // namespace Gecko

Source/Core/Core/HLE/HLE.cpp

@@ -60,7 +60,9 @@ static const SPatch OSPatches[] = {
     {"___blank", HLE_OS::HLE_GeneralDebugPrint, HLE_HOOK_REPLACE, HLE_TYPE_DEBUG},
     {"__write_console", HLE_OS::HLE_write_console, HLE_HOOK_REPLACE,
      HLE_TYPE_DEBUG},  // used by sysmenu (+more?)
-    {"GeckoCodehandler", HLE_Misc::HLEGeckoCodehandler, HLE_HOOK_START, HLE_TYPE_GENERIC},
+    {"GeckoCodehandler", HLE_Misc::GeckoCodeHandlerICacheFlush, HLE_HOOK_START, HLE_TYPE_GENERIC},
+    {"GeckoHandlerReturnTrampoline", HLE_Misc::GeckoReturnTrampoline, HLE_HOOK_REPLACE,
+     HLE_TYPE_GENERIC},
 };
 
 static const SPatch OSBreakPoints[] = {

Source/Core/Core/HLE/HLE_Misc.cpp

@@ -2,22 +2,17 @@
 // Licensed under GPLv2+
 // Refer to the license.txt file included.
 
-#include <cmath>
-#include <string>
-
-#include "Common/CommonTypes.h"
-#include "Core/ConfigManager.h"
-#include "Core/GeckoCode.h"
 #include "Core/HLE/HLE_Misc.h"
+#include "Common/CommonTypes.h"
+#include "Common/Logging/Log.h"
+#include "Common/MsgHandler.h"
+#include "Core/GeckoCode.h"
 #include "Core/HW/CPU.h"
 #include "Core/Host.h"
-#include "Core/PowerPC/PPCCache.h"
 #include "Core/PowerPC/PowerPC.h"
 
 namespace HLE_Misc
 {
-static std::string args;
-
 // If you just want to kill a function, one of the three following are usually appropriate.
 // According to the PPC ABI, the return value is always in r3.
 void UnimplementedFunction()
@@ -40,7 +35,7 @@ void HBReload()
   Host_Message(WM_USER_STOP);
 }
 
-void HLEGeckoCodehandler()
+void GeckoCodeHandlerICacheFlush()
 {
   // Work around the codehandler not properly invalidating the icache, but
   // only the first few frames.
@@ -61,4 +56,22 @@ void HLEGeckoCodehandler()
 
   PowerPC::ppcState.iCache.Reset();
 }
+
+// Because Dolphin messes around with the CPU state instead of patching the game binary, we
+// need a way to branch into the GCH from an arbitrary PC address. Branching is easy, returning
+// back is the hard part. This HLE function acts as a trampoline that restores the original LR, SP,
+// and PC before the magic, invisible BL instruction happened.
+void GeckoReturnTrampoline()
+{
+  // Stack frame is built in GeckoCode.cpp, Gecko::RunCodeHandler.
+  u32 SP = GPR(1);
+  GPR(1) = PowerPC::HostRead_U32(SP + 8);
+  NPC = PowerPC::HostRead_U32(SP + 12);
+  LR = PowerPC::HostRead_U32(SP + 16);
+  for (int i = 0; i < 14; ++i)
+  {
+    riPS0(i) = PowerPC::HostRead_U64(SP + 24 + 2 * i * sizeof(u64));
+    riPS1(i) = PowerPC::HostRead_U64(SP + 24 + (2 * i + 1) * sizeof(u64));
+  }
+}
 }

Source/Core/Core/HLE/HLE_Misc.h

@@ -9,5 +9,6 @@ namespace HLE_Misc
 void HLEPanicAlert();
 void UnimplementedFunction();
 void HBReload();
-void HLEGeckoCodehandler();
+void GeckoCodeHandlerICacheFlush();
+void GeckoReturnTrampoline();
 }

Source/Core/Core/PatchEngine.cpp

@@ -216,7 +216,7 @@ void ApplyFramePatches()
   ApplyPatches(onFrame);
 
   // Run the Gecko code handler
-  Gecko::RunCodeHandler();
+  Gecko::RunCodeHandler(oldMSR);
   ActionReplay::RunAllActive();
   MSR = oldMSR;
 }

Source/Core/Core/PowerPC/MMU.cpp

@@ -565,6 +565,11 @@ u32 HostRead_U32(const u32 address)
   return var;
 }
 
+u64 HostRead_U64(const u32 address)
+{
+  return ReadFromHardware<FLAG_NO_EXCEPTION, u64>(address);
+}
+
 void HostWrite_U8(const u8 var, const u32 address)
 {
   WriteToHardware<FLAG_NO_EXCEPTION, u8>(address, var);

Source/Core/Core/PowerPC/PowerPC.h

@@ -209,6 +209,7 @@ void UpdatePerformanceMonitor(u32 cycles, u32 num_load_stores, u32 num_fp_inst);
 u8 HostRead_U8(const u32 address);
 u16 HostRead_U16(const u32 address);
 u32 HostRead_U32(const u32 address);
+u64 HostRead_U64(const u32 address);
 u32 HostRead_Instruction(const u32 address);
 
 void HostWrite_U8(const u8 var, const u32 address);