GS/HW: Further improve alpha calculation when alpha testing

pcsx2/GS/GSState.cpp

@@ -3970,24 +3970,24 @@ void GSState::CorrectATEAlphaMinMax(const u32 atst, const int aref)
 	switch (atst)
 	{
 		case ATST_LESS:
-			amax = std::min(amax, aref - 1);
-			amin = std::min(amin, amax);
+			amin = std::min(amin, std::max(aref - 1, amin));
+			amax = std::min(amax, std::max(aref - 1, amin));
 			break;
 		case ATST_LEQUAL:
-			amax = std::min(amax, aref);
-			amin = std::min(amin, aref);
+			amin = std::min(amin, std::max(aref, amin));
+			amax = std::min(amax, std::max(aref, amin));
 			break;
 		case ATST_EQUAL:
 			amax = aref;
 			amin = aref;
 			break;
 		case ATST_GEQUAL:
-			amax = std::max(amax, aref);
-			amin = std::max(amin, aref);
+			amax = std::max(amax, std::min(aref, amax));
+			amin = std::max(amin, std::min(aref, amax));
 			break;
 		case ATST_GREATER:
-			amax = std::max(amax, aref + 1);
-			amin = std::max(amin, aref + 1);
+			amax = std::max(amax, std::min(aref + 1, amax));
+			amin = std::max(amin, std::min(aref + 1, amax));
 			break;
 		default:
 			break;

pcsx2/GS/Renderers/HW/GSRendererHW.cpp

@@ -5060,16 +5060,15 @@ __ri void GSRendererHW::DrawPrims(GSTextureCache::Target* rt, GSTextureCache::Ta
 		}
 	}
 
+	const int fail_type = m_cached_ctx.TEST.GetAFAIL(m_cached_ctx.FRAME.PSM);
+	const int aref = static_cast<int>(m_cached_ctx.TEST.AREF);
+	if (m_cached_ctx.TEST.ATE && ((fail_type != AFAIL_FB_ONLY && fail_type != AFAIL_RGB_ONLY) || !PRIM->ABE || !IsUsingAsInBlend()))
+		CorrectATEAlphaMinMax(m_cached_ctx.TEST.ATST, aref);
+
 	// Blend
 	int blend_alpha_min = 0, blend_alpha_max = 255;
 	if (rt)
 	{
-		{
-			const int fail_type = m_cached_ctx.TEST.GetAFAIL(m_cached_ctx.FRAME.PSM);
-			const int aref = static_cast<int>(m_cached_ctx.TEST.AREF);
-			if (m_cached_ctx.TEST.ATE && ((fail_type != AFAIL_FB_ONLY) || ((fail_type == AFAIL_RGB_ONLY) && (m_conf.ps.dst_fmt != GSLocalMemory::PSM_FMT_32))))
-				CorrectATEAlphaMinMax(m_cached_ctx.TEST.ATST, aref);
-		}
 
 		blend_alpha_min = rt->m_alpha_min;
 		blend_alpha_max = rt->m_alpha_max;