GS/HW: Implement HDR RTA.

Double the alpha so we get proper Ad for hw blending.

bin/resources/shaders/dx11/convert.fx

@@ -129,6 +129,22 @@ PS_OUTPUT ps_hdr_resolve(PS_INPUT input)
 	return output;
 }
 
+PS_OUTPUT ps_hdr_rta_init(PS_INPUT input)
+{
+	PS_OUTPUT output;
+	float4 value = sample_c(input.t);
+	output.c = float4(round(value.rgb * 255) / 65535, trunc(value.a * 255 + 0.1) / 128);
+	return output;
+}
+
+PS_OUTPUT ps_hdr_rta_resolve(PS_INPUT input)
+{
+	PS_OUTPUT output;
+	float4 value = sample_c(input.t);
+	output.c = float4(float3(uint3(value.rgb * 65535.5) & 255) / 255, (value.a * 128) / 255);
+	return output;
+}
+
 uint ps_convert_float32_32bits(PS_INPUT input) : SV_Target0
 {
 	// Convert a FLOAT32 depth texture into a 32 bits UINT texture

bin/resources/shaders/dx11/tfx.fx

@@ -818,7 +818,7 @@ void ps_color_clamp_wrap(inout float3 C)
 		// In 16 bits format, only 5 bits of color are used. It impacts shadows computation of Castlevania
 		if (PS_DST_FMT == FMT_16 && PS_BLEND_MIX == 0)
 			C = (float3)((int3)C & (int3)0xF8);
-		else if (PS_COLCLIP == 1 || PS_HDR == 1)
+		else if (PS_COLCLIP == 1 || PS_HDR != 0)
 			C = (float3)((int3)C & (int3)0xFF);
 	}
 }
@@ -1069,7 +1069,7 @@ PS_OUTPUT ps_main(PS_INPUT input)
 	ps_fbmask(C, input.p.xy);
 
 #if !PS_NO_COLOR
-	output.c0 = PS_HDR ? float4(C.rgb / 65535.0f, C.a / 255.0f) : C / 255.0f;
+	output.c0 = PS_HDR == 2 ? float4(C.rgb / 65535.0f, C.a / 128.0f) : PS_HDR == 1 ? float4(C.rgb / 65535.0f, C.a / 255.0f) : C / 255.0f;
 #if !PS_NO_COLOR1
 	output.c1 = alpha_blend;
 #endif

bin/resources/shaders/opengl/convert.glsl

@@ -329,6 +329,22 @@ void ps_hdr_resolve()
 }
 #endif
 
+#ifdef ps_hdr_rta_init
+void ps_hdr_rta_init()
+{
+	vec4 value = sample_c();
+	SV_Target0 = vec4(round(value.rgb * 255.0f) / 65535.0f, trunc(value.a * 255 + 0.1) / 128);
+}
+#endif
+
+#ifdef ps_hdr_rta_resolve
+void ps_hdr_rta_resolve()
+{
+	vec4 value = sample_c();
+	SV_Target0 = vec4(vec3(uvec3(value.rgb * 65535.0f) & 255u) / 255.0f, (value.a * 128) / 255);
+}
+#endif
+
 #ifdef ps_convert_clut_4
 uniform uvec3 offset;
 uniform float scale;

bin/resources/shaders/opengl/tfx_fs.glsl

@@ -762,7 +762,7 @@ void ps_color_clamp_wrap(inout vec3 C)
 #if PS_DST_FMT == FMT_16 && PS_BLEND_MIX == 0
 	// In 16 bits format, only 5 bits of colors are used. It impacts shadows computation of Castlevania
 	C = vec3(ivec3(C) & ivec3(0xF8));
-#elif PS_COLCLIP == 1 || PS_HDR == 1
+#elif PS_COLCLIP == 1 || PS_HDR != 0
 	C = vec3(ivec3(C) & ivec3(0xFF));
 #endif
 
@@ -1063,7 +1063,9 @@ void ps_main()
 	ps_fbmask(C);
 
 #if !PS_NO_COLOR
-	#if PS_HDR == 1
+	#if PS_HDR == 2
+		SV_Target0 = vec4(C.rgb / 65535.0f, C.a / 128.0f);
+	#elif PS_HDR == 1
 		SV_Target0 = vec4(C.rgb / 65535.0f, C.a / 255.0f);
 	#else
 		SV_Target0 = C / 255.0f;

bin/resources/shaders/vulkan/convert.glsl

@@ -108,6 +108,22 @@ void ps_hdr_resolve()
 }
 #endif
 
+#ifdef ps_hdr_rta_init
+void ps_hdr_rta_init()
+{
+	vec4 value = sample_c(v_tex);
+	o_col0 = vec4(roundEven(value.rgb * 255.0f) / 65535.0f, trunc(value.a * 255 + 0.1) / 128);
+}
+#endif
+
+#ifdef ps_hdr_rta_resolve
+void ps_hdr_rta_resolve()
+{
+	vec4 value = sample_c(v_tex);
+	o_col0 = vec4(vec3(uvec3(value.rgb * 65535.5f) & 255u) / 255.0f, (value.a * 128) / 255);
+}
+#endif
+
 #ifdef ps_convert_float32_32bits
 void ps_convert_float32_32bits()
 {

bin/resources/shaders/vulkan/tfx.glsl

@@ -1014,7 +1014,7 @@ void ps_color_clamp_wrap(inout vec3 C)
 #if PS_DST_FMT == FMT_16 && PS_BLEND_MIX == 0
 	// In 16 bits format, only 5 bits of colors are used. It impacts shadows computation of Castlevania
 	C = vec3(ivec3(C) & ivec3(0xF8));
-#elif PS_COLCLIP == 1 || PS_HDR == 1
+#elif PS_COLCLIP == 1 || PS_HDR != 0
 	C = vec3(ivec3(C) & ivec3(0xFF));
 #endif
 
@@ -1294,7 +1294,9 @@ void main()
 	ps_fbmask(C);
 
 	#if !PS_NO_COLOR
-		#if PS_HDR == 1
+		#if PS_HDR == 2
+			o_col0 = vec4(C.rgb / 65535.0f, C.a / 128.0f);
+		#elif PS_HDR == 1
 			o_col0 = vec4(C.rgb / 65535.0f, C.a / 255.0f);
 		#else
 			o_col0 = C / 255.0f;

pcsx2/GS/Renderers/Common/GSDevice.cpp

@@ -28,6 +28,8 @@ const char* shaderName(ShaderConvert value)
 		case ShaderConvert::DATM_0:                 return "ps_datm0";
 		case ShaderConvert::HDR_INIT:               return "ps_hdr_init";
 		case ShaderConvert::HDR_RESOLVE:            return "ps_hdr_resolve";
+		case ShaderConvert::HDR_RTA_INIT:           return "ps_hdr_rta_init";
+		case ShaderConvert::HDR_RTA_RESOLVE:        return "ps_hdr_rta_resolve";
 		case ShaderConvert::TRANSPARENCY_FILTER:    return "ps_filter_transparency";
 		case ShaderConvert::FLOAT32_TO_16_BITS:     return "ps_convert_float32_32bits";
 		case ShaderConvert::FLOAT32_TO_32_BITS:     return "ps_convert_float32_32bits";

pcsx2/GS/Renderers/Common/GSDevice.h

@@ -21,6 +21,8 @@ enum class ShaderConvert
 	DATM_0,
 	HDR_INIT,
 	HDR_RESOLVE,
+	HDR_RTA_INIT,
+	HDR_RTA_RESOLVE,
 	TRANSPARENCY_FILTER,
 	FLOAT32_TO_16_BITS,
 	FLOAT32_TO_32_BITS,
@@ -314,7 +316,7 @@ struct alignas(16) GSHWDrawConfig
 				u32 fixed_one_a : 1;
 				u32 blend_hw    : 2;
 				u32 a_masked    : 1;
-				u32 hdr         : 1;
+				u32 hdr         : 2;
 				u32 colclip     : 1;
 				u32 blend_mix   : 2;
 				u32 round_inv   : 1; // Blending will invert the value, so rounding needs to go the other way

pcsx2/GS/Renderers/DX11/GSDevice11.cpp

@@ -2511,7 +2511,7 @@ void GSDevice11::RenderHW(GSHWDrawConfig& config)
 
 		// Warning: StretchRect must be called before BeginScene otherwise
 		// vertices will be overwritten. Trust me you don't want to do that.
-		StretchRect(config.rt, sRect, hdr_rt, dRect, ShaderConvert::HDR_INIT, false);
+		StretchRect(config.rt, sRect, hdr_rt, dRect, config.ps.hdr == 2 ? ShaderConvert::HDR_RTA_INIT : ShaderConvert::HDR_INIT, false);
 		g_perfmon.Put(GSPerfMon::TextureCopies, 1);
 	}
 
@@ -2660,7 +2660,7 @@ void GSDevice11::RenderHW(GSHWDrawConfig& config)
 		const GSVector2i size = config.rt->GetSize();
 		const GSVector4 dRect(config.drawarea);
 		const GSVector4 sRect = dRect / GSVector4(size.x, size.y).xyxy();
-		StretchRect(hdr_rt, sRect, config.rt, dRect, ShaderConvert::HDR_RESOLVE, false);
+		StretchRect(hdr_rt, sRect, config.rt, dRect, config.ps.hdr == 2 ? ShaderConvert::HDR_RTA_RESOLVE : ShaderConvert::HDR_RESOLVE, false);
 		g_perfmon.Put(GSPerfMon::TextureCopies, 1);
 		Recycle(hdr_rt);
 	}

pcsx2/GS/Renderers/DX12/GSDevice12.cpp

@@ -2476,6 +2476,23 @@ bool GSDevice12::CompileConvertPipelines()
 				D3D12::SetObjectName(arr[ds].get(), TinyString::from_fmt("HDR {}/copy pipeline (ds={})", is_setup ? "setup" : "finish", ds));
 			}
 		}
+		else if (i == ShaderConvert::HDR_RTA_INIT || i == ShaderConvert::HDR_RTA_RESOLVE)
+		{
+			const bool is_setup = i == ShaderConvert::HDR_RTA_INIT;
+			std::array<ComPtr<ID3D12PipelineState>, 2>& arr = is_setup ? m_hdr_rta_setup_pipelines : m_hdr_rta_finish_pipelines;
+			for (u32 ds = 0; ds < 2; ds++)
+			{
+				pxAssert(!arr[ds]);
+
+				gpb.SetRenderTarget(0, is_setup ? DXGI_FORMAT_R16G16B16A16_UNORM : DXGI_FORMAT_R8G8B8A8_UNORM);
+				gpb.SetDepthStencilFormat(ds ? DXGI_FORMAT_D32_FLOAT_S8X24_UINT : DXGI_FORMAT_UNKNOWN);
+				arr[ds] = gpb.Create(m_device.get(), m_shader_cache, false);
+				if (!arr[ds])
+					return false;
+
+				D3D12::SetObjectName(arr[ds].get(), TinyString::from_fmt("HDR RTA {}/copy pipeline (ds={})", is_setup ? "setup" : "finish", ds));
+			}
+		}
 	}
 
 	for (u32 datm = 0; datm < 2; datm++)
@@ -2701,6 +2718,8 @@ void GSDevice12::DestroyResources()
 	m_convert = {};
 	m_hdr_setup_pipelines = {};
 	m_hdr_finish_pipelines = {};
+	m_hdr_rta_setup_pipelines = {};
+	m_hdr_rta_finish_pipelines = {};
 	m_date_image_setup_pipelines = {};
 	m_fxaa_pipeline.reset();
 	m_shadeboost_pipeline.reset();
@@ -3875,7 +3894,7 @@ void GSDevice12::RenderHW(GSHWDrawConfig& config)
 	if (hdr_rt && config.rt->GetState() == GSTexture::State::Dirty)
 	{
 		SetUtilityTexture(static_cast<GSTexture12*>(config.rt), m_point_sampler_cpu);
-		SetPipeline(m_hdr_setup_pipelines[pipe.ds].get());
+		SetPipeline(config.ps.hdr == 2 ? m_hdr_rta_setup_pipelines[pipe.ds].get() : m_hdr_setup_pipelines[pipe.ds].get());
 
 		const GSVector4 sRect(GSVector4(render_area) / GSVector4(rtsize.x, rtsize.y).xyxy());
 		DrawStretchRect(sRect, GSVector4(render_area), rtsize);
@@ -3954,7 +3973,7 @@ void GSDevice12::RenderHW(GSHWDrawConfig& config)
 			draw_rt->GetClearColor(), 0.0f, 0);
 
 		const GSVector4 sRect(GSVector4(render_area) / GSVector4(rtsize.x, rtsize.y).xyxy());
-		SetPipeline(m_hdr_finish_pipelines[pipe.ds].get());
+		SetPipeline(config.ps.hdr == 2 ? m_hdr_rta_finish_pipelines[pipe.ds].get() : m_hdr_finish_pipelines[pipe.ds].get());
 		SetUtilityTexture(hdr_rt, m_point_sampler_cpu);
 		DrawStretchRect(sRect, GSVector4(render_area), rtsize);
 		g_perfmon.Put(GSPerfMon::TextureCopies, 1);

pcsx2/GS/Renderers/DX12/GSDevice12.h

@@ -317,6 +317,8 @@ class GSDevice12 final : public GSDevice
 	std::array<ComPtr<ID3D12PipelineState>, NUM_INTERLACE_SHADERS> m_interlace{};
 	std::array<ComPtr<ID3D12PipelineState>, 2> m_hdr_setup_pipelines{}; // [depth]
 	std::array<ComPtr<ID3D12PipelineState>, 2> m_hdr_finish_pipelines{}; // [depth]
+	std::array<ComPtr<ID3D12PipelineState>, 2> m_hdr_rta_setup_pipelines{}; // [depth]
+	std::array<ComPtr<ID3D12PipelineState>, 2> m_hdr_rta_finish_pipelines{}; // [depth]
 	std::array<std::array<ComPtr<ID3D12PipelineState>, 2>, 2> m_date_image_setup_pipelines{}; // [depth][datm]
 	ComPtr<ID3D12PipelineState> m_fxaa_pipeline;
 	ComPtr<ID3D12PipelineState> m_shadeboost_pipeline;

pcsx2/GS/Renderers/HW/GSRendererHW.cpp

@@ -4035,6 +4035,7 @@ void GSRendererHW::EmulateBlending(int rt_alpha_min, int rt_alpha_max, bool& DAT
 	// Color clip
 	if (COLCLAMP.CLAMP == 0)
 	{
+		const bool hdr_rta = m_conf.ps.blend_c == 1 && !m_cached_ctx.TEST.DATE && !blend_ad_alpha_masked && rt_alpha_max <= 128;
 		bool free_colclip = false;
 		if (features.framebuffer_fetch)
 			free_colclip = true;
@@ -4072,6 +4073,11 @@ void GSRendererHW::EmulateBlending(int rt_alpha_min, int rt_alpha_max, bool& DAT
 			GL_INS("COLCLIP SW mode ENABLED");
 			m_conf.ps.colclip = 1;
 		}
+		else if (hdr_rta)
+		{
+			GL_INS("COLCLIP HDR RTA mode ENABLED");
+			m_conf.ps.hdr = 2;
+		}
 		else
 		{
 			GL_INS("COLCLIP HDR mode ENABLED");
@@ -4288,7 +4294,7 @@ void GSRendererHW::EmulateBlending(int rt_alpha_min, int rt_alpha_max, bool& DAT
 
 			m_conf.ps.blend_hw = 2;
 		}
-		else if (blend_flag & BLEND_HW_CLR3)
+		else if (m_conf.ps.hdr != 2 && blend_flag & BLEND_HW_CLR3)
 		{
 			m_conf.ps.blend_hw = 3;
 		}

pcsx2/GS/Renderers/Metal/GSDeviceMTL.mm

@@ -1037,12 +1037,14 @@ static void OnMainThread(Fn&& fn)
 	// FS Triangle Pipelines
 	pdesc.colorAttachments[0].pixelFormat = ConvertPixelFormat(GSTexture::Format::Color);
 	m_hdr_resolve_pipeline = MakePipeline(pdesc, fs_triangle, LoadShader(@"ps_hdr_resolve"), @"HDR Resolve");
+	m_hdr_rta_resolve_pipeline = MakePipeline(pdesc, fs_triangle, LoadShader(@"ps_hdr_rta_resolve"), @"HDR RTA Resolve");
 	m_fxaa_pipeline = MakePipeline(pdesc, fs_triangle, LoadShader(@"ps_fxaa"), @"fxaa");
 	m_shadeboost_pipeline = MakePipeline(pdesc, fs_triangle, LoadShader(@"ps_shadeboost"), @"shadeboost");
 	m_clut_pipeline[0] = MakePipeline(pdesc, fs_triangle, LoadShader(@"ps_convert_clut_4"), @"4-bit CLUT Update");
 	m_clut_pipeline[1] = MakePipeline(pdesc, fs_triangle, LoadShader(@"ps_convert_clut_8"), @"8-bit CLUT Update");
 	pdesc.colorAttachments[0].pixelFormat = ConvertPixelFormat(GSTexture::Format::HDRColor);
 	m_hdr_init_pipeline = MakePipeline(pdesc, fs_triangle, LoadShader(@"ps_hdr_init"), @"HDR Init");
+	m_hdr_rta_init_pipeline = MakePipeline(pdesc, fs_triangle, LoadShader(@"ps_hdr_rta_init"), @"HDR RTA Init");
 	m_hdr_clear_pipeline = MakePipeline(pdesc, fs_triangle, LoadShader(@"ps_clear"), @"HDR Clear");
 	pdesc.colorAttachments[0].pixelFormat = MTLPixelFormatInvalid;
 	pdesc.stencilAttachmentPixelFormat = MTLPixelFormatDepth32Float_Stencil8;
@@ -2145,8 +2147,8 @@ static bool usesStencil(GSHWDrawConfig::DestinationAlphaMode dstalpha)
 		switch (config.rt->GetState())
 		{
 			case GSTexture::State::Dirty:
-				BeginRenderPass(@"HDR Init", hdr_rt, MTLLoadActionDontCare, nullptr, MTLLoadActionDontCare);
-				RenderCopy(config.rt, m_hdr_init_pipeline, config.drawarea);
+				BeginRenderPass(config.ps.hdr == 2 ? @"HDR RTA Init" : @"HDR Init", hdr_rt, MTLLoadActionDontCare, nullptr, MTLLoadActionDontCare);
+				RenderCopy(config.rt, config.ps.hdr == 2 ? : m_hdr_rta_init_pipeline : m_hdr_init_pipeline, config.drawarea);
 				g_perfmon.Put(GSPerfMon::TextureCopies, 1);
 				break;
 
@@ -2210,8 +2212,8 @@ static bool usesStencil(GSHWDrawConfig::DestinationAlphaMode dstalpha)
 
 	if (hdr_rt)
 	{
-		BeginRenderPass(@"HDR Resolve", config.rt, MTLLoadActionLoad, nullptr, MTLLoadActionDontCare);
-		RenderCopy(hdr_rt, m_hdr_resolve_pipeline, config.drawarea);
+		BeginRenderPass(config.ps.hdr == 2 ? @"HDR RTA Resolve" : @"HDR Resolve", config.rt, MTLLoadActionLoad, nullptr, MTLLoadActionDontCare);
+		RenderCopy(hdr_rt, config.ps.hdr == 2 ? m_hdr_rta_resolve_pipeline : m_hdr_resolve_pipeline, config.drawarea);
 		g_perfmon.Put(GSPerfMon::TextureCopies, 1);
 
 		Recycle(hdr_rt);

pcsx2/GS/Renderers/Metal/convert.metal

@@ -110,12 +110,24 @@ fragment float4 ps_hdr_init(float4 p [[position]], DirectReadTextureIn<float> te
 	return float4(round(in.rgb * 255.f) / 65535.f, in.a);
 }
 
+fragment float4 ps_hdr_rta_init(float4 p [[position]], DirectReadTextureIn<float> tex)
+{
+	float4 in = tex.read(p);
+	return float4(round(in.rgb * 255.f) / 65535.f, trunc(in.a * 255.f + 0.1f) / 128.f);
+}
+
 fragment float4 ps_hdr_resolve(float4 p [[position]], DirectReadTextureIn<float> tex)
 {
 	float4 in = tex.read(p);
 	return float4(float3(uint3(in.rgb * 65535.5f) & 255) / 255.f, in.a);
 }
 
+fragment float4 ps_hdr_rta_resolve(float4 p [[position]], DirectReadTextureIn<float> tex)
+{
+	float4 in = tex.read(p);
+	return float4(float3(uint3(in.rgb * 65535.5f) & 255) / 255.f, (in.a * 128.f) / 255.f);
+}
+
 fragment float4 ps_filter_transparency(ConvertShaderData data [[stage_in]], ConvertPSRes res)
 {
 	float4 c = res.sample(data.t);

pcsx2/GS/Renderers/Metal/tfx.metal

@@ -40,7 +40,7 @@ constant uint PS_BLEND_C            [[function_constant(GSMTLConstantIndex_PS_BL
 constant uint PS_BLEND_D            [[function_constant(GSMTLConstantIndex_PS_BLEND_D)]];
 constant uint PS_BLEND_HW           [[function_constant(GSMTLConstantIndex_PS_BLEND_HW)]];
 constant bool PS_A_MASKED           [[function_constant(GSMTLConstantIndex_PS_A_MASKED)]];
-constant bool PS_HDR                [[function_constant(GSMTLConstantIndex_PS_HDR)]];
+constant uint PS_HDR                [[function_constant(GSMTLConstantIndex_PS_HDR)]];
 constant bool PS_COLCLIP            [[function_constant(GSMTLConstantIndex_PS_COLCLIP)]];
 constant uint PS_BLEND_MIX          [[function_constant(GSMTLConstantIndex_PS_BLEND_MIX)]];
 constant bool PS_ROUND_INV          [[function_constant(GSMTLConstantIndex_PS_ROUND_INV)]];
@@ -1121,7 +1121,7 @@ struct PSMain
 		ps_fbmask(C);
 
 		if (PS_COLOR0)
-			out.c0 = PS_HDR ? float4(C.rgb / 65535.f, C.a / 255.f) : C / 255.f;
+			out.c0 = PS_HDR == 2 ? float4(C.rgb / 65535.f, C.a / 128.f) : PS_HDR == 1 ? float4(C.rgb / 65535.f, C.a / 255.f): C / 255.f;
 		if (PS_COLOR0 && PS_ONLY_ALPHA)
 			out.c0.rgb = 0;
 		if (PS_COLOR1)

pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp

@@ -2411,9 +2411,9 @@ void GSDeviceOGL::RenderHW(GSHWDrawConfig& config)
 		hdr_rt = CreateRenderTarget(rtsize.x, rtsize.y, GSTexture::Format::HDRColor, false);
 		OMSetRenderTargets(hdr_rt, config.ds, &config.scissor);
 
-		GSVector4 dRect(config.drawarea);
+		const GSVector4 dRect(config.drawarea);
 		const GSVector4 sRect = dRect / GSVector4(rtsize.x, rtsize.y).xyxy();
-		StretchRect(config.rt, sRect, hdr_rt, dRect, ShaderConvert::HDR_INIT, false);
+		StretchRect(config.rt, sRect, hdr_rt, dRect, config.ps.hdr == 2 ? ShaderConvert::HDR_RTA_INIT : ShaderConvert::HDR_INIT, false);
 	}
 	else if (config.require_one_barrier && !m_features.texture_barrier)
 	{
@@ -2619,10 +2619,10 @@ void GSDeviceOGL::RenderHW(GSHWDrawConfig& config)
 
 	if (hdr_rt)
 	{
-		GSVector2i size = config.rt->GetSize();
-		GSVector4 dRect(config.drawarea);
+		const GSVector2i size = config.rt->GetSize();
+		const GSVector4 dRect(config.drawarea);
 		const GSVector4 sRect = dRect / GSVector4(size.x, size.y).xyxy();
-		StretchRect(hdr_rt, sRect, config.rt, dRect, ShaderConvert::HDR_RESOLVE, false);
+		StretchRect(hdr_rt, sRect, config.rt, dRect, config.ps.hdr == 2 ? ShaderConvert::HDR_RTA_RESOLVE : ShaderConvert::HDR_RESOLVE, false);
 
 		Recycle(hdr_rt);
 	}