GS/HW: Make sure if it's a shuffle when checking sources alpha min max.

pcsx2/GS/Renderers/HW/GSRendererHW.cpp

@@ -2576,7 +2576,7 @@ void GSRendererHW::Draw()
 			// We don't know the alpha range of direct sources when we first tried to optimize the alpha test.
 			// Moving the texture lookup before the ATST optimization complicates things a lot, so instead,
 			// recompute it, and everything derived from it again if it changes.
-			if (src->m_valid_alpha_minmax)
+			if (!IsPossibleChannelShuffle() && src->m_valid_alpha_minmax)
 			{
 				CalcAlphaMinMax(src->m_alpha_minmax.first, src->m_alpha_minmax.second);