diff --git a/pcsx2/Memory.cpp b/pcsx2/Memory.cpp
index 830dc1efbcd15a..1e09b6718b77ab 100644
--- a/pcsx2/Memory.cpp
+++ b/pcsx2/Memory.cpp
@@ -1134,9 +1134,6 @@ void memReset()
 	//vtlb_VMap(0x00000000,0x00000000,0x20000000);
 	//vtlb_VMapUnmap(0x20000000,0x60000000);
 
-	// update extra memory mode before memory is mapped
-	memSetExtraMemMode(EmuConfig.Cpu.ExtraMemory);
-
 	memMapPhy();
 	memMapVUmicro();
 	memMapKernelMem();
diff --git a/pcsx2/VMManager.cpp b/pcsx2/VMManager.cpp
index 2cb267113e9df6..f9123e05c6f7f7 100644
--- a/pcsx2/VMManager.cpp
+++ b/pcsx2/VMManager.cpp
@@ -1376,6 +1376,7 @@ bool VMManager::Initialize(VMBootParameters boot_params)
 
 	s_cpu_implementation_changed = false;
 	UpdateCPUImplementations();
+	memSetExtraMemMode(EmuConfig.Cpu.ExtraMemory);
 	Internal::ClearCPUExecutionCaches();
 	FPControlRegister::SetCurrent(EmuConfig.Cpu.FPUFPCR);
 	memBindConditionalHandlers();
@@ -1616,6 +1617,7 @@ void VMManager::Reset()
 	if (elf_was_changed)
 		HandleELFChange(false);
 
+	memSetExtraMemMode(EmuConfig.Cpu.ExtraMemory);
 	Internal::ClearCPUExecutionCaches();
 	memBindConditionalHandlers();
 	SysMemory::Reset();
@@ -2611,14 +2613,6 @@ void VMManager::Internal::ELFLoadingOnCPUThread(std::string elf_path)
 		Patch::ReloadPatches(s_disc_serial, 0, false, false, false, true);
 		ApplyCoreSettings();
 	}
-
-	if (CHECK_EXTRAMEM)
-	{
-		// If we're using extra memory, we might need to reallocate CPU memory.
-		// This must be done here to ensure there is enough RAM to load the ELF,
-		// because it may have segments that will be mapped into high memory.
-		Cpu->Reset();
-	}
 }
 
 void VMManager::Internal::EntryPointCompilingOnCPUThread()