From d7059e05baebd3973ad0f79dec6cf589e9fba075 Mon Sep 17 00:00:00 2001
From: lightningterror <18107717+lightningterror@users.noreply.github.com>
Date: Sat, 9 Mar 2024 20:51:52 +0100
Subject: [PATCH] GS/HW: Adjust blend mix for impossible blend.

 Since we can't do Cd*(As + 1) - Cs*As in hw blend what we can do is adjust the Cs value that will be subtracted, this way we can get a better result in hw blend. Result is still wrong but less wrong than before.
---
 bin/resources/shaders/dx11/tfx.fx        | 10 ++++++++--
 bin/resources/shaders/opengl/tfx_fs.glsl | 10 ++++++++--
 bin/resources/shaders/vulkan/tfx.glsl    | 10 ++++++++--
 pcsx2/GS/Renderers/Metal/tfx.metal       | 10 ++++++++--
 4 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/bin/resources/shaders/dx11/tfx.fx b/bin/resources/shaders/dx11/tfx.fx
index c4ae8a939be08c..e6e5f06dd1ee77 100644
--- a/bin/resources/shaders/dx11/tfx.fx
+++ b/bin/resources/shaders/dx11/tfx.fx
@@ -862,7 +862,7 @@ void ps_blend(inout float4 Color, inout float4 As_rgba, float2 pos_xy)
 		// As/Af clamp alpha for Blend mix
 		// We shouldn't clamp blend mix with blend hw 1 as we want alpha higher
 		float C_clamped = C;
-		if (PS_BLEND_MIX > 0 && PS_BLEND_HW != 1)
+		if (PS_BLEND_MIX > 0 && PS_BLEND_HW != 1 && PS_BLEND_HW != 2)
 			C_clamped = min(C_clamped, 1.0f);
 
 		if (PS_BLEND_A == PS_BLEND_B)
@@ -895,12 +895,18 @@ void ps_blend(inout float4 Color, inout float4 As_rgba, float2 pos_xy)
 		}
 		else if (PS_BLEND_HW == 2)
 		{
+			// Since we can't do Cd*(As + 1) - Cs*As in hw blend
+			// what we can do is adjust the Cs value that will be
+			// subtracted, this way we can get a better result in hw blend.
+			// Result is still wrong but less wrong than before.
+			float division_alpha = 1.0f + (C / 2.0f);
+			Color.rgb /= (float3)division_alpha;
 			// Compensate slightly for Cd*(As + 1) - Cs*As.
 			// The initial factor we chose is 1 (0.00392)
 			// as that is the minimum color Cd can be,
 			// then we multiply by alpha to get the minimum
 			// blended value it can be.
-			float color_compensate = 1.0f * (C + 1.0f);
+			float color_compensate = (C > 0.0f) ? 1.0f * (C + 1.0f) : 0.0f;
 			Color.rgb -= (float3)color_compensate;
 		}
 		else if (PS_BLEND_HW == 3)
diff --git a/bin/resources/shaders/opengl/tfx_fs.glsl b/bin/resources/shaders/opengl/tfx_fs.glsl
index d6fbda5090f4bf..e9e45a115a37f9 100644
--- a/bin/resources/shaders/opengl/tfx_fs.glsl
+++ b/bin/resources/shaders/opengl/tfx_fs.glsl
@@ -844,7 +844,7 @@ float As = As_rgba.a;
 	// As/Af clamp alpha for Blend mix
 	// We shouldn't clamp blend mix with blend hw 1 as we want alpha higher
 	float C_clamped = C;
-#if PS_BLEND_MIX > 0 && PS_BLEND_HW != 1
+#if PS_BLEND_MIX > 0 && PS_BLEND_HW != 1 && PS_BLEND_HW != 2
 	C_clamped = min(C_clamped, 1.0f);
 #endif
 
@@ -876,12 +876,18 @@ float As = As_rgba.a;
 	vec3 alpha_compensate = max(vec3(1.0f), Color.rgb / vec3(255.0f));
 	As_rgba.rgb -= alpha_compensate;
 #elif PS_BLEND_HW == 2
+	// Since we can't do Cd*(As + 1) - Cs*As in hw blend
+	// what we can do is adjust the Cs value that will be
+	// subtracted, this way we can get a better result in hw blend.
+	// Result is still wrong but less wrong than before.
+	float division_alpha = 1.0f + (C / 2.0f);
+	Color.rgb /= vec3(division_alpha);
 	// Compensate slightly for Cd*(As + 1) - Cs*As.
 	// The initial factor we chose is 1 (0.00392)
 	// as that is the minimum color Cd can be,
 	// then we multiply by alpha to get the minimum
 	// blended value it can be.
-	float color_compensate = 1.0f * (C + 1.0f);
+	float color_compensate = (C > 0.0f) ? 1.0f * (C + 1.0f) : 0.0f;
 	Color.rgb -= vec3(color_compensate);
 #elif PS_BLEND_HW == 3
 	// As, Ad or Af clamped.
diff --git a/bin/resources/shaders/vulkan/tfx.glsl b/bin/resources/shaders/vulkan/tfx.glsl
index 36695b4b88294b..69d2f54770c3bf 100644
--- a/bin/resources/shaders/vulkan/tfx.glsl
+++ b/bin/resources/shaders/vulkan/tfx.glsl
@@ -1098,7 +1098,7 @@ void ps_blend(inout vec4 Color, inout vec4 As_rgba)
 		// As/Af clamp alpha for Blend mix
 		// We shouldn't clamp blend mix with blend hw 1 as we want alpha higher
 		float C_clamped = C;
-		#if PS_BLEND_MIX > 0 && PS_BLEND_HW != 1
+		#if PS_BLEND_MIX > 0 && PS_BLEND_HW != 1 && PS_BLEND_HW != 2
 			C_clamped = min(C_clamped, 1.0f);
 		#endif
 
@@ -1130,12 +1130,18 @@ void ps_blend(inout vec4 Color, inout vec4 As_rgba)
 			vec3 alpha_compensate = max(vec3(1.0f), Color.rgb / vec3(255.0f));
 			As_rgba.rgb -= alpha_compensate;
 		#elif PS_BLEND_HW == 2
+			// Since we can't do Cd*(As + 1) - Cs*As in hw blend
+			// what we can do is adjust the Cs value that will be
+			// subtracted, this way we can get a better result in hw blend.
+			// Result is still wrong but less wrong than before.
+			float division_alpha = 1.0f + (C / 2.0f);
+			Color.rgb /= vec3(division_alpha);
 			// Compensate slightly for Cd*(As + 1) - Cs*As.
 			// The initial factor we chose is 1 (0.00392)
 			// as that is the minimum color Cd can be,
 			// then we multiply by alpha to get the minimum
 			// blended value it can be.
-			float color_compensate = 1.0f * (C + 1.0f);
+			float color_compensate = (C > 0.0f) ? 1.0f * (C + 1.0f) : 0.0f;
 			Color.rgb -= vec3(color_compensate);
 		#elif PS_BLEND_HW == 3
 			// As, Ad or Af clamped.
diff --git a/pcsx2/GS/Renderers/Metal/tfx.metal b/pcsx2/GS/Renderers/Metal/tfx.metal
index 65c7f79a678e54..ea2c6680695319 100644
--- a/pcsx2/GS/Renderers/Metal/tfx.metal
+++ b/pcsx2/GS/Renderers/Metal/tfx.metal
@@ -927,7 +927,7 @@ struct PSMain
 			// As/Af clamp alpha for Blend mix
 			// We shouldn't clamp blend mix with blend hw 1 as we want alpha higher
 			float C_clamped = C;
-			if (PS_BLEND_MIX > 0 && PS_BLEND_HW != 1)
+			if (PS_BLEND_MIX > 0 && PS_BLEND_HW != 1 && PS_BLEND_HW != 2)
 				C_clamped = min(C_clamped, 1.f);
 
 			if (PS_BLEND_A == PS_BLEND_B)
@@ -960,12 +960,18 @@ struct PSMain
 			}
 			else if (PS_BLEND_HW == 2)
 			{
+				// Since we can't do Cd*(As + 1) - Cs*As in hw blend
+				// what we can do is adjust the Cs value that will be
+				// subtracted, this way we can get a better result in hw blend.
+				// Result is still wrong but less wrong than before.
+				float division_alpha = 1.f + (C / 2.f);
+				Color.rgb /= float3(division_alpha);
 				// Compensate slightly for Cd*(As + 1) - Cs*As.
 				// The initial factor we chose is 1 (0.00392)
 				// as that is the minimum color Cd can be,
 				// then we multiply by alpha to get the minimum
 				// blended value it can be.
-				float color_compensate = 1.f * (C + 1.f);
+				float color_compensate = (C > 0.f) ? 1.0f * (C + 1.f) : 0.f;
 				Color.rgb -= float3(color_compensate);
 			}
 			else if (PS_BLEND_HW == 3)