diff --git a/Source/Core/VideoCommon/VertexLoader.h b/Source/Core/VideoCommon/VertexLoader.h
index 73738c86b8..4be2f41a65 100644
--- a/Source/Core/VideoCommon/VertexLoader.h
+++ b/Source/Core/VideoCommon/VertexLoader.h
@@ -22,7 +22,6 @@ public:
   VertexLoader(const TVtxDesc& vtx_desc, const VAT& vtx_attr);
 
   int RunVertices(DataReader src, DataReader dst, int count) override;
-  bool IsInitialized() override { return true; }  // This vertex loader supports all formats
   // They are used for the communication with the loader functions
   float m_posScale;
   float m_tcScale[8];
diff --git a/Source/Core/VideoCommon/VertexLoaderARM64.cpp b/Source/Core/VideoCommon/VertexLoaderARM64.cpp
index 8dc396b47c..4030a50808 100644
--- a/Source/Core/VideoCommon/VertexLoaderARM64.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderARM64.cpp
@@ -53,9 +53,6 @@ alignas(16) static const float scale_factors[] = {
 VertexLoaderARM64::VertexLoaderARM64(const TVtxDesc& vtx_desc, const VAT& vtx_att)
     : VertexLoaderBase(vtx_desc, vtx_att), m_float_emit(this)
 {
-  if (!IsInitialized())
-    return;
-
   AllocCodeSpace(4096);
   ClearCodeSpace();
   GenerateVertexLoader();
diff --git a/Source/Core/VideoCommon/VertexLoaderARM64.h b/Source/Core/VideoCommon/VertexLoaderARM64.h
index 6b18b59c0e..2b2787709b 100644
--- a/Source/Core/VideoCommon/VertexLoaderARM64.h
+++ b/Source/Core/VideoCommon/VertexLoaderARM64.h
@@ -19,7 +19,6 @@ public:
   VertexLoaderARM64(const TVtxDesc& vtx_desc, const VAT& vtx_att);
 
 protected:
-  bool IsInitialized() override { return true; }
   int RunVertices(DataReader src, DataReader dst, int count) override;
 
 private:
diff --git a/Source/Core/VideoCommon/VertexLoaderBase.cpp b/Source/Core/VideoCommon/VertexLoaderBase.cpp
index 6a8c6498c0..84d7eb43c0 100644
--- a/Source/Core/VideoCommon/VertexLoaderBase.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderBase.cpp
@@ -34,31 +34,23 @@ public:
                      const TVtxDesc& vtx_desc, const VAT& vtx_attr)
       : VertexLoaderBase(vtx_desc, vtx_attr), a(std::move(a_)), b(std::move(b_))
   {
-    m_initialized = a && b && a->IsInitialized() && b->IsInitialized();
-
-    if (m_initialized)
+    ASSERT(a && b);
+    if (a->m_VertexSize == b->m_VertexSize && a->m_native_components == b->m_native_components &&
+        a->m_native_vtx_decl.stride == b->m_native_vtx_decl.stride)
     {
-      m_initialized = a->m_VertexSize == b->m_VertexSize &&
-                      a->m_native_components == b->m_native_components &&
-                      a->m_native_vtx_decl.stride == b->m_native_vtx_decl.stride;
-
-      if (m_initialized)
-      {
-        m_VertexSize = a->m_VertexSize;
-        m_native_components = a->m_native_components;
-        memcpy(&m_native_vtx_decl, &a->m_native_vtx_decl, sizeof(PortableVertexDeclaration));
-      }
-      else
-      {
-        ERROR_LOG_FMT(VIDEO, "Can't compare vertex loaders that expect different vertex formats!");
-        ERROR_LOG_FMT(VIDEO, "a: m_VertexSize {}, m_native_components {:#010x}, stride {}",
-                      a->m_VertexSize, a->m_native_components, a->m_native_vtx_decl.stride);
-        ERROR_LOG_FMT(VIDEO, "b: m_VertexSize {}, m_native_components {:#010x}, stride {}",
-                      b->m_VertexSize, b->m_native_components, b->m_native_vtx_decl.stride);
-      }
+      m_VertexSize = a->m_VertexSize;
+      m_native_components = a->m_native_components;
+      memcpy(&m_native_vtx_decl, &a->m_native_vtx_decl, sizeof(PortableVertexDeclaration));
+    }
+    else
+    {
+      ERROR_LOG_FMT(VIDEO, "Can't compare vertex loaders that expect different vertex formats!");
+      ERROR_LOG_FMT(VIDEO, "a: m_VertexSize {}, m_native_components {:#010x}, stride {}",
+                    a->m_VertexSize, a->m_native_components, a->m_native_vtx_decl.stride);
+      ERROR_LOG_FMT(VIDEO, "b: m_VertexSize {}, m_native_components {:#010x}, stride {}",
+                    b->m_VertexSize, b->m_native_components, b->m_native_vtx_decl.stride);
     }
   }
-  ~VertexLoaderTester() override {}
   int RunVertices(DataReader src, DataReader dst, int count) override
   {
     buffer_a.resize(count * a->m_native_vtx_decl.stride + 4);
@@ -81,21 +73,17 @@ public:
                std::min(count_a, count_b) * m_native_vtx_decl.stride))
     {
       ERROR_LOG_FMT(VIDEO,
-                    "The two vertex loaders have loaded different data "
-                    "(guru meditation {:#010x}, {:#010x}, {:#010x}, {:#010x}, {:#010x}).",
-                    m_VtxDesc.low.Hex, m_VtxDesc.high.Hex, m_VtxAttr.g0.Hex, m_VtxAttr.g1.Hex,
-                    m_VtxAttr.g2.Hex);
+                    "The two vertex loaders have loaded different data.  Configuration:"
+                    "\nVertex desc:\n{}\n\nVertex attr:\n{}",
+                    m_VtxDesc, m_VtxAttr);
     }
 
     memcpy(dst.GetPointer(), buffer_a.data(), count_a * m_native_vtx_decl.stride);
     m_numLoadedVertices += count;
     return count_a;
   }
-  bool IsInitialized() override { return m_initialized; }
 
 private:
-  bool m_initialized;
-
   std::unique_ptr<VertexLoaderBase> a;
   std::unique_ptr<VertexLoaderBase> b;
 
@@ -106,33 +94,29 @@ private:
 std::unique_ptr<VertexLoaderBase> VertexLoaderBase::CreateVertexLoader(const TVtxDesc& vtx_desc,
                                                                        const VAT& vtx_attr)
 {
-  std::unique_ptr<VertexLoaderBase> loader;
+  std::unique_ptr<VertexLoaderBase> loader = nullptr;
 
   //#define COMPARE_VERTEXLOADERS
 
-#if defined(COMPARE_VERTEXLOADERS) && defined(_M_X86_64)
-  // first try: Any new VertexLoader vs the old one
-  loader = std::make_unique<VertexLoaderTester>(
-      std::make_unique<VertexLoader>(vtx_desc, vtx_attr),     // the software one
-      std::make_unique<VertexLoaderX64>(vtx_desc, vtx_attr),  // the new one to compare
-      vtx_desc, vtx_attr);
-  if (loader->IsInitialized())
-    return loader;
-#elif defined(_M_X86_64)
+#if defined(_M_X86_64)
   loader = std::make_unique<VertexLoaderX64>(vtx_desc, vtx_attr);
-  if (loader->IsInitialized())
-    return loader;
 #elif defined(_M_ARM_64)
   loader = std::make_unique<VertexLoaderARM64>(vtx_desc, vtx_attr);
-  if (loader->IsInitialized())
-    return loader;
 #endif
 
-  // last try: The old VertexLoader
-  loader = std::make_unique<VertexLoader>(vtx_desc, vtx_attr);
-  if (loader->IsInitialized())
-    return loader;
+  // Use the software loader as a fallback
+  // (not currently applicable, as both VertexLoaderX64 and VertexLoaderARM64
+  // are always usable, but if a loader that only works on some CPUs is created
+  // then this fallback would be used)
+  if (!loader)
+    loader = std::make_unique<VertexLoader>(vtx_desc, vtx_attr);
 
-  PanicAlertFmt("No Vertex Loader found.");
-  return nullptr;
+#if defined(COMPARE_VERTEXLOADERS)
+  return std::make_unique<VertexLoaderTester>(
+      std::make_unique<VertexLoader>(vtx_desc, vtx_attr),  // the software one
+      std::move(loader),                                   // the new one to compare
+      vtx_desc, vtx_attr);
+#else
+  return loader;
+#endif
 }
diff --git a/Source/Core/VideoCommon/VertexLoaderBase.h b/Source/Core/VideoCommon/VertexLoaderBase.h
index bc57d34b4f..2c787479b6 100644
--- a/Source/Core/VideoCommon/VertexLoaderBase.h
+++ b/Source/Core/VideoCommon/VertexLoaderBase.h
@@ -65,8 +65,6 @@ public:
   virtual ~VertexLoaderBase() {}
   virtual int RunVertices(DataReader src, DataReader dst, int count) = 0;
 
-  virtual bool IsInitialized() = 0;
-
   // per loader public state
   int m_VertexSize = 0;  // number of bytes of a raw GC vertex
   PortableVertexDeclaration m_native_vtx_decl{};
diff --git a/Source/Core/VideoCommon/VertexLoaderX64.cpp b/Source/Core/VideoCommon/VertexLoaderX64.cpp
index 9e1b056b97..e6d94693b8 100644
--- a/Source/Core/VideoCommon/VertexLoaderX64.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderX64.cpp
@@ -45,9 +45,6 @@ static OpArg MPIC(const void* ptr)
 VertexLoaderX64::VertexLoaderX64(const TVtxDesc& vtx_desc, const VAT& vtx_att)
     : VertexLoaderBase(vtx_desc, vtx_att)
 {
-  if (!IsInitialized())
-    return;
-
   AllocCodeSpace(4096);
   ClearCodeSpace();
   GenerateVertexLoader();
diff --git a/Source/Core/VideoCommon/VertexLoaderX64.h b/Source/Core/VideoCommon/VertexLoaderX64.h
index 4b3029066c..7b91ad6cf8 100644
--- a/Source/Core/VideoCommon/VertexLoaderX64.h
+++ b/Source/Core/VideoCommon/VertexLoaderX64.h
@@ -18,7 +18,6 @@ public:
   VertexLoaderX64(const TVtxDesc& vtx_desc, const VAT& vtx_att);
 
 protected:
-  bool IsInitialized() override { return true; }
   int RunVertices(DataReader src, DataReader dst, int count) override;
 
 private: