diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp b/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp
index 134b18b496..2812e422bf 100644
--- a/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp
+++ b/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp
@@ -3,7 +3,6 @@
 
 #include "VideoCommon/Assets/CustomAssetLoader.h"
 
-#include "Common/Logging/Log.h"
 #include "Common/MemoryUtil.h"
 #include "VideoCommon/Assets/CustomAssetLibrary.h"
 
@@ -48,19 +47,22 @@ void CustomAssetLoader::Init()
   m_asset_load_thread.Reset("Custom Asset Loader", [this](std::weak_ptr<CustomAsset> asset) {
     if (auto ptr = asset.lock())
     {
+      if (m_memory_exceeded)
+        return;
+
       if (ptr->Load())
       {
         std::lock_guard lk(m_asset_load_lock);
         const std::size_t asset_memory_size = ptr->GetByteSizeInMemory();
-        if (m_max_memory_available >= m_total_bytes_loaded + asset_memory_size)
+        m_total_bytes_loaded += asset_memory_size;
+        m_assets_to_monitor.try_emplace(ptr->GetAssetId(), ptr);
+        if (m_total_bytes_loaded > m_max_memory_available)
         {
-          m_total_bytes_loaded += asset_memory_size;
-          m_assets_to_monitor.try_emplace(ptr->GetAssetId(), ptr);
-        }
-        else
-        {
-          ERROR_LOG_FMT(VIDEO, "Failed to load asset {} because there was not enough memory.",
+          ERROR_LOG_FMT(VIDEO,
+                        "Asset memory exceeded with asset '{}', future assets won't load until "
+                        "memory is available.",
                         ptr->GetAssetId());
+          m_memory_exceeded = true;
         }
       }
     }
diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLoader.h b/Source/Core/VideoCommon/Assets/CustomAssetLoader.h
index 920f62e830..ebf1e01456 100644
--- a/Source/Core/VideoCommon/Assets/CustomAssetLoader.h
+++ b/Source/Core/VideoCommon/Assets/CustomAssetLoader.h
@@ -10,6 +10,7 @@
 #include <thread>
 
 #include "Common/Flag.h"
+#include "Common/Logging/Log.h"
 #include "Common/WorkQueueThread.h"
 #include "VideoCommon/Assets/CustomAsset.h"
 #include "VideoCommon/Assets/MaterialAsset.h"
@@ -67,6 +68,11 @@ private:
         std::lock_guard lk(m_asset_load_lock);
         m_total_bytes_loaded -= a->GetByteSizeInMemory();
         m_assets_to_monitor.erase(a->GetAssetId());
+        if (m_max_memory_available >= m_total_bytes_loaded && m_memory_exceeded)
+        {
+          INFO_LOG_FMT(VIDEO, "Asset memory went below limit, new assets can begin loading.");
+          m_memory_exceeded = false;
+        }
       }
       delete a;
     });
@@ -85,6 +91,7 @@ private:
 
   std::size_t m_total_bytes_loaded = 0;
   std::size_t m_max_memory_available = 0;
+  std::atomic_bool m_memory_exceeded = false;
 
   std::map<CustomAssetLibrary::AssetID, std::weak_ptr<CustomAsset>> m_assets_to_monitor;