Merge pull request #9436 from shuffle2/asan

msvc: enable asan compat

Source/Core/Core/MemTools.cpp

@@ -9,6 +9,7 @@
 #include <cstring>
 #include <vector>
 
+#include "Common/Assert.h"
 #include "Common/CommonFuncs.h"
 #include "Common/CommonTypes.h"
 #include "Common/MsgHandler.h"
@@ -28,30 +29,32 @@ namespace EMM
 {
 #ifdef _WIN32
 
+static PVOID s_veh_handle;
+
 static LONG NTAPI Handler(PEXCEPTION_POINTERS pPtrs)
 {
   switch (pPtrs->ExceptionRecord->ExceptionCode)
   {
   case EXCEPTION_ACCESS_VIOLATION:
   {
-    int accessType = (int)pPtrs->ExceptionRecord->ExceptionInformation[0];
-    if (accessType == 8)  // Rule out DEP
+    ULONG_PTR access_type = pPtrs->ExceptionRecord->ExceptionInformation[0];
+    if (access_type == 8)  // Rule out DEP
     {
-      return (DWORD)EXCEPTION_CONTINUE_SEARCH;
+      return EXCEPTION_CONTINUE_SEARCH;
     }
 
     // virtual address of the inaccessible data
-    uintptr_t badAddress = (uintptr_t)pPtrs->ExceptionRecord->ExceptionInformation[1];
-    CONTEXT* ctx = pPtrs->ContextRecord;
+    uintptr_t fault_address = (uintptr_t)pPtrs->ExceptionRecord->ExceptionInformation[1];
+    SContext* ctx = pPtrs->ContextRecord;
 
-    if (JitInterface::HandleFault(badAddress, ctx))
+    if (JitInterface::HandleFault(fault_address, ctx))
     {
-      return (DWORD)EXCEPTION_CONTINUE_EXECUTION;
+      return EXCEPTION_CONTINUE_EXECUTION;
     }
     else
     {
       // Let's not prevent debugging.
-      return (DWORD)EXCEPTION_CONTINUE_SEARCH;
+      return EXCEPTION_CONTINUE_SEARCH;
     }
   }
 
@@ -84,18 +87,17 @@ static LONG NTAPI Handler(PEXCEPTION_POINTERS pPtrs)
 
 void InstallExceptionHandler()
 {
-  // Make sure this is only called once per process execution
-  // Instead, could make a Uninstall function, but whatever..
-  static bool handlerInstalled = false;
-  if (handlerInstalled)
+  if (s_veh_handle)
     return;
 
-  AddVectoredExceptionHandler(TRUE, Handler);
-  handlerInstalled = true;
+  s_veh_handle = AddVectoredExceptionHandler(TRUE, Handler);
+  ASSERT(s_veh_handle);
 }
 
 void UninstallExceptionHandler()
 {
+  ULONG status = RemoveVectoredExceptionHandler(s_veh_handle);
+  ASSERT(status);
 }
 
 #elif defined(__APPLE__) && !defined(USE_SIGACTION_ON_APPLE)

Source/UnitTests/Core/PageFaultTest.cpp

@@ -49,6 +49,17 @@ public:
       m_post_unprotect_time;
 };
 
+#ifdef _MSC_VER
+#define ASAN_DISABLE __declspec(no_sanitize_address)
+#else
+#define ASAN_DISABLE
+#endif
+
+static void ASAN_DISABLE perform_invalid_access(void* data)
+{
+  *(volatile int*)data = 5;
+}
+
 TEST(PageFault, PageFault)
 {
   EMM::InstallExceptionHandler();
@@ -61,7 +72,7 @@ TEST(PageFault, PageFault)
   pfjit.m_data = data;
 
   auto start = std::chrono::high_resolution_clock::now();
-  *(volatile int*)data = 5;
+  perform_invalid_access(data);
   auto end = std::chrono::high_resolution_clock::now();
 
 #define AS_NS(diff)                                                                                \

Source/VSProps/Base.props

@@ -133,6 +133,9 @@
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
       <Optimization>Disabled</Optimization>
     </ClCompile>
+    <ClCompile Condition="'$(Configuration)'=='Debug' And '$(EnableASAN)'=='true'">
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+    </ClCompile>
     <!--ClCompile Release-->
     <ClCompile Condition="'$(Configuration)'=='Release'">
       <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>

Source/VSProps/Configuration.Base.props

@@ -4,6 +4,11 @@
     <PlatformToolset>v142</PlatformToolset>
     <CharacterSet>Unicode</CharacterSet>
     <PreferredToolArchitecture>x64</PreferredToolArchitecture>
+    <!-- To use ASAN, just uncomment this. For simplicity, you should run VS/windbg/etc
+    (including the built executables themselves) after using vcvarsall or similar to setup
+    environment, as ASAN needs access to libs and executables in the toolchain paths.
+    -->
+    <!--<EnableASAN>true</EnableASAN>-->
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
     <UseDebugLibraries>true</UseDebugLibraries>