D3D: Move exclusive mode switching to UI thread.

This prevents deadlocks when switching to exclusive mode.
And it also allows the CPU thread to block until we've completed the switch.

Source/Core/DolphinWX/Frame.cpp

@@ -1131,28 +1131,12 @@ void CFrame::OnMouse(wxMouseEvent& event)
 
 void CFrame::DoFullscreen(bool enable_fullscreen)
 {
-  if (g_Config.bExclusiveMode && Core::GetState() == Core::CORE_PAUSE)
-  {
-    // A responsive renderer is required for exclusive fullscreen, but the
-    // renderer can only respond in the running state. Therefore we ignore
-    // fullscreen switches if we are in exclusive fullscreen, but the
-    // renderer is not running.
-    // TODO: Allow the renderer to switch fullscreen modes while paused.
-    return;
-  }
-
   ToggleDisplayMode(enable_fullscreen);
 
-  if (enable_fullscreen)
-  {
-    m_RenderFrame->ShowFullScreen(true, wxFULLSCREEN_ALL);
-  }
-  else if (!g_Config.bExclusiveMode)
-  {
-    // Exiting exclusive fullscreen should be done from a Renderer callback.
-    // Therefore we don't exit fullscreen from here if we are in exclusive mode.
-    m_RenderFrame->ShowFullScreen(false, wxFULLSCREEN_ALL);
-  }
+  if (!enable_fullscreen && g_renderer)
+    g_renderer->SetFullscreen(false);
+
+  m_RenderFrame->ShowFullScreen(enable_fullscreen, wxFULLSCREEN_ALL);
 
   if (SConfig::GetInstance().bRenderToMain)
   {
@@ -1200,9 +1184,10 @@ void CFrame::DoFullscreen(bool enable_fullscreen)
   else
   {
     m_RenderFrame->Raise();
-  }
 
-  g_Config.bFullscreen = enable_fullscreen;
+    if (enable_fullscreen && g_renderer)
+      g_renderer->SetFullscreen(true);
+  }
 }
 
 const CGameListCtrl* CFrame::GetGameListCtrl() const

Source/Core/DolphinWX/FrameTools.cpp

@@ -787,12 +787,6 @@ void CFrame::DoStop()
       // waiting on inputs
       bool should_pause = !NetPlayDialog::GetNetPlayClient();
 
-      // If exclusive fullscreen is not enabled then we can pause the emulation
-      // before we've exited fullscreen. If not then we need to exit fullscreen first.
-      should_pause =
-          should_pause && (!RendererIsFullscreen() || !g_Config.ExclusiveFullscreenEnabled() ||
-                           SConfig::GetInstance().bRenderToMain);
-
       if (should_pause)
       {
         Core::SetState(Core::CORE_PAUSE);

Source/Core/DolphinWX/Globals.h

@@ -292,7 +292,6 @@ enum
   IDM_WINDOW_SIZE_REQUEST,
   IDM_STOPPED,
   IDM_HOST_MESSAGE,
-  IDM_FULLSCREEN_REQUEST,
 
   // Used for Host_ConnectWiimote()
   IDM_FORCE_CONNECT_WIIMOTE1,

Source/Core/VideoBackends/D3D/Render.cpp

@@ -5,6 +5,7 @@
 #include <cinttypes>
 #include <cmath>
 #include <memory>
+#include <mutex>
 #include <string>
 #include <strsafe.h>
 #include <unordered_map>
@@ -43,6 +44,9 @@ namespace DX11
 static u32 s_last_multisamples = 1;
 static bool s_last_stereo_mode = false;
 static bool s_last_xfb_mode = false;
+static bool s_last_exclusive_mode = false;
+
+static std::mutex s_critical_fullscreen;
 
 static Television s_television;
 
@@ -851,8 +855,6 @@ void Renderer::SwapImpl(u32 xfbAddr, u32 fbWidth, u32 fbStride, u32 fbHeight,
   SetWindowSize(fbStride, fbHeight);
 
   const bool windowResized = CheckForResize();
-  const bool fullscreen = g_ActiveConfig.bFullscreen && !g_ActiveConfig.bBorderlessFullscreen &&
-                          !SConfig::GetInstance().bRenderToMain;
 
   bool xfbchanged = s_last_xfb_mode != g_ActiveConfig.bUseRealXFB;
 
@@ -869,33 +871,19 @@ void Renderer::SwapImpl(u32 xfbAddr, u32 fbWidth, u32 fbStride, u32 fbHeight,
   // Flip/present backbuffer to frontbuffer here
   D3D::Present();
 
-  // Check exclusive fullscreen state
-  bool exclusive_mode, fullscreen_changed = false;
-  if (SUCCEEDED(D3D::GetFullscreenState(&exclusive_mode)))
+  // Check if we need to regain exclusive mode
   {
-    if (fullscreen && !exclusive_mode)
-    {
-      if (g_Config.bExclusiveMode)
-        OSD::AddMessage("Lost exclusive fullscreen.");
+    std::lock_guard<std::mutex> lk(s_critical_fullscreen);
 
-      // Exclusive fullscreen is enabled in the configuration, but we're
-      // not in exclusive mode. Either exclusive fullscreen was turned on
-      // or the render frame lost focus. When the render frame is in focus
-      // we can apply exclusive mode.
-      fullscreen_changed = Host_RendererHasFocus();
-
-      g_Config.bExclusiveMode = false;
-    }
-    else if (!fullscreen && exclusive_mode)
-    {
-      // Exclusive fullscreen is disabled, but we're still in exclusive mode.
-      fullscreen_changed = true;
-    }
+    bool exclusive_mode = false;
+    D3D::GetFullscreenState(&exclusive_mode);
+    if (s_last_exclusive_mode && !exclusive_mode && Host_RendererHasFocus())
+      D3D::SetFullscreenState(true);
   }
 
   // Resize the back buffers NOW to avoid flickering
   if (CalculateTargetSize(s_backbuffer_width, s_backbuffer_height) || xfbchanged || windowResized ||
-      fullscreen_changed || s_last_efb_scale != g_ActiveConfig.iEFBScale ||
+      s_last_efb_scale != g_ActiveConfig.iEFBScale ||
       s_last_multisamples != g_ActiveConfig.iMultisamples ||
       s_last_stereo_mode != (g_ActiveConfig.iStereoMode > 0))
   {
@@ -903,23 +891,8 @@ void Renderer::SwapImpl(u32 xfbAddr, u32 fbWidth, u32 fbStride, u32 fbHeight,
     s_last_multisamples = g_ActiveConfig.iMultisamples;
     PixelShaderCache::InvalidateMSAAShaders();
 
-    if (windowResized || fullscreen_changed)
+    if (windowResized)
     {
-      // Apply fullscreen state
-      if (fullscreen_changed)
-      {
-        g_Config.bExclusiveMode = fullscreen;
-
-        if (fullscreen)
-          OSD::AddMessage("Entered exclusive fullscreen.");
-
-        D3D::SetFullscreenState(fullscreen);
-
-        // If fullscreen is disabled we can safely notify the UI to exit fullscreen.
-        if (!g_ActiveConfig.bFullscreen)
-          Host_RequestFullscreen(false);
-      }
-
       // TODO: Aren't we still holding a reference to the back buffer right now?
       D3D::Reset();
       SAFE_RELEASE(s_screenshot_texture);
@@ -1291,4 +1264,11 @@ void Renderer::BlitScreen(TargetRectangle src, TargetRectangle dst, D3DTexture2D
   }
 }
 
+void Renderer::SetFullscreen(bool enable_fullscreen)
+{
+  std::lock_guard<std::mutex> lk(s_critical_fullscreen);
+  s_last_exclusive_mode = enable_fullscreen;
+  D3D::SetFullscreenState(enable_fullscreen);
+}
+
 }  // namespace DX11

Source/Core/VideoBackends/D3D/Render.h

@@ -25,6 +25,7 @@ public:
   void SetSamplerState(int stage, int texindex, bool custom_tex) override;
   void SetInterlacingMode() override;
   void SetViewport() override;
+  void SetFullscreen(bool enable_fullscreen) override;
 
   // TODO: Fix confusing names (see ResetAPIState and RestoreAPIState)
   void ApplyState(bool bUseDstAlpha) override;

Source/Core/VideoCommon/RenderBase.h

@@ -72,6 +72,7 @@ public:
   virtual void SetSamplerState(int stage, int texindex, bool custom_tex) {}
   virtual void SetInterlacingMode() {}
   virtual void SetViewport() {}
+  virtual void SetFullscreen(bool enable_fullscreen) {}
   virtual void ApplyState(bool bUseDstAlpha) {}
   virtual void RestoreState() {}
   virtual void ResetAPIState() {}