[WIP] GS/HW: Initial work implementing RT in RT support

bin/resources/GameIndex.yaml

@@ -1967,6 +1967,7 @@ SCAJ-20125:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.
@@ -1977,6 +1978,7 @@ SCAJ-20126:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.
@@ -2454,6 +2456,7 @@ SCAJ-20199:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.
@@ -4149,6 +4152,7 @@ SCED-53538:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.
@@ -5769,6 +5773,7 @@ SCES-53202:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.
@@ -7212,6 +7217,7 @@ SCKA-20049:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.
@@ -7435,6 +7441,7 @@ SCKA-20081:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.
@@ -57353,6 +57360,7 @@ SLPS-25510:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1 # Fixes vertical lines.
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.
@@ -60518,6 +60526,7 @@ SLPS-73223:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1 # Fixes vertical lines.
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.
@@ -66516,6 +66525,7 @@ SLUS-21059:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1 # Fixes vertical lines.
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.
@@ -67060,6 +67070,7 @@ SLUS-21160:
   clampModes:
     eeClampMode: 2 # Fixes camera and stops constant coin noises on Pirates Cove.
   gsHWFixes:
+    textureInsideRT: 1 # Fixes heat haze half screen problem.
     alignSprite: 1 # Fixes vertical lines.
     halfPixelOffset: 4 # Align post.
     nativeScaling: 1 # Fixes depth of field effect.

bin/resources/shaders/dx11/tfx.fx

@@ -1123,11 +1123,8 @@ PS_OUTPUT ps_main(PS_INPUT input)
 		{
 			if (PS_PROCESS_BA == SHUFFLE_READWRITE && PS_PROCESS_RG == SHUFFLE_READWRITE)
 			{
-				C.rb = C.br;
-				float g_temp = C.g;
-
-				C.g = C.a;
-				C.a = g_temp;
+				C.br = C.rb;				
+				C.ag = C.ga;
 			}
 			else if(PS_PROCESS_BA & SHUFFLE_READ)
 			{

bin/resources/shaders/opengl/tfx_fs.glsl

@@ -1086,11 +1086,8 @@ void ps_main()
 			C.ga = vec2(float((denorm_c.g >> 6) | ((denorm_c.b >> 3) << 2) | (denorm_TA.x & 0x80u)));
 	#elif PS_SHUFFLE_ACROSS
 		#if(PS_PROCESS_BA == SHUFFLE_READWRITE && PS_PROCESS_RG == SHUFFLE_READWRITE)
-			C.rb = C.br;
-			float g_temp = C.g;
-
-			C.g = C.a;
-			C.a = g_temp;
+			C.br = C.rb;				
+			C.ag = C.ga;
 		#elif(PS_PROCESS_BA & SHUFFLE_READ)
 			C.rb = C.bb;
 			C.ga = C.aa;

bin/resources/shaders/vulkan/tfx.glsl

@@ -945,7 +945,7 @@ vec4 ps_color()
 	vec4 T = sample_color(st);
 #endif
 
-	#if PS_SHUFFLE && !PS_READ16_SRC && !PS_SHUFFLE_SAME
+	#if PS_SHUFFLE && !PS_READ16_SRC && !PS_SHUFFLE_SAME && !(PS_PROCESS_BA == SHUFFLE_READWRITE && PS_PROCESS_RG == SHUFFLE_READWRITE)
 		uvec4 denorm_c_before = uvec4(T);
 		#if (PS_PROCESS_BA & SHUFFLE_READ)
 			T.r = float((denorm_c_before.b << 3) & 0xF8u);
@@ -1320,7 +1320,7 @@ void main()
 	ps_blend(C, alpha_blend);
 
 #if PS_SHUFFLE
-		#if !PS_READ16_SRC && !PS_SHUFFLE_SAME
+		#if !PS_READ16_SRC && !PS_SHUFFLE_SAME && !(PS_PROCESS_BA == SHUFFLE_READWRITE && PS_PROCESS_RG == SHUFFLE_READWRITE)
 			uvec4 denorm_c_after = uvec4(C);
 			#if (PS_PROCESS_BA & SHUFFLE_READ)
 				C.b = float(((denorm_c_after.r >> 3) & 0x1Fu) | ((denorm_c_after.g << 2) & 0xE0u));
@@ -1350,11 +1350,8 @@ void main()
 		// Write RB part. Mask will take care of the correct destination
 		#elif PS_SHUFFLE_ACROSS
 			#if(PS_PROCESS_BA == SHUFFLE_READWRITE && PS_PROCESS_RG == SHUFFLE_READWRITE)
-				C.rb = C.br;
-				float g_temp = C.g;
-
-				C.g = C.a;
-				C.a = g_temp;
+				C.br = C.rb;				
+				C.ag = C.ga;
 			#elif(PS_PROCESS_BA & SHUFFLE_READ)
 				C.rb = C.bb;
 				C.ga = C.aa;

pcsx2/GS/GSState.cpp

@@ -467,7 +467,8 @@ void GSState::DumpVertices(const std::string& filename)
 		file << std::setfill('0') << std::setw(3) << unsigned(v.RGBAQ.R) << DEL;
 		file << std::setfill('0') << std::setw(3) << unsigned(v.RGBAQ.G) << DEL;
 		file << std::setfill('0') << std::setw(3) << unsigned(v.RGBAQ.B) << DEL;
-		file << std::setfill('0') << std::setw(3) << unsigned(v.RGBAQ.A);
+		file << std::setfill('0') << std::setw(3) << unsigned(v.RGBAQ.A) << DEL;
+		file << "FOG: " << std::setfill('0') << std::setw(3) << unsigned(v.FOG);
 		file << std::endl;
 	}
 
@@ -1674,7 +1675,8 @@ void GSState::FlushPrim()
 			Console.Warning("GS: Possible invalid draw, Frame PSM %x ZPSM %x", m_context->FRAME.PSM, m_context->ZBUF.PSM);
 		}
 #endif
-
+		// Update scissor, it may have been modified by a previous draw
+		m_env.CTXT[PRIM->CTXT].UpdateScissor();
 		m_vt.Update(m_vertex.buff, m_index.buff, m_vertex.tail, m_index.tail, GSUtil::GetPrimClass(PRIM->PRIM));
 
 		// Texel coordinate rounding
@@ -3094,6 +3096,16 @@ __forceinline bool GSState::IsAutoFlushDraw(u32 prim)
 	if (!(GSUtil::GetChannelMask(m_context->TEX0.PSM) & GSUtil::GetChannelMask(m_context->FRAME.PSM, m_context->FRAME.FBMSK | ~(GSLocalMemory::m_psm[m_context->FRAME.PSM].fmsk))))
 		return false;
 
+	// Try to detect shuffles, because these will not autoflush, they by design clash.
+	if (GSLocalMemory::m_psm[m_context->FRAME.PSM].bpp == 16 && GSLocalMemory::m_psm[m_context->TEX0.PSM].bpp == 16)
+	{
+		// Pretty confident here...
+		GSVertex* buffer = &m_vertex.buff[0];
+		const bool const_spacing = std::abs(buffer[m_index.buff[0]].U - buffer[m_index.buff[0]].XYZ.X) == std::abs(m_v.U - m_v.XYZ.X) && std::abs(buffer[m_index.buff[1]].XYZ.X - buffer[m_index.buff[0]].XYZ.X) < 64;
+
+		if (const_spacing)
+			return false;
+	}
 	const u32 frame_mask = GSLocalMemory::m_psm[m_context->FRAME.PSM].fmsk;
 	const bool frame_hit = m_context->FRAME.Block() == m_context->TEX0.TBP0 && !(m_context->TEST.ATE && m_context->TEST.ATST == 0 && m_context->TEST.AFAIL == 2) && ((m_context->FRAME.FBMSK & frame_mask) != frame_mask);
 	// There's a strange behaviour we need to test on a PS2 here, if the FRAME is a Z format, like Powerdrome something swaps over, and it seems Alpha Fail of "FB Only" writes to the Z.. it's odd.
@@ -3859,7 +3871,8 @@ GSState::TextureMinMaxResult GSState::GetTextureMinMax(GIFRegTEX0 TEX0, GIFRegCL
 		const GSVector2 grad(uv_range / pos_range);
 		// Adjust texture range when sprites get scissor clipped. Since we linearly interpolate, this
 		// optimization doesn't work when perspective correction is enabled.
-		if (m_vt.m_primclass == GS_SPRITE_CLASS && PRIM->FST == 1 && m_primitive_covers_without_gaps != NoGapsType::GapsFound)
+		// Allowing for quads when the gradiant is 1. It's not guaranteed (would need to check the grandient on each vector), but should be close enough.
+		if ((m_vt.m_primclass == GS_SPRITE_CLASS || (m_vt.m_primclass == GS_TRIANGLE_CLASS && TrianglesAreQuads(false) && grad.x == 1.0f && grad.y == 1.0f)) && m_primitive_covers_without_gaps != NoGapsType::GapsFound)
 		{
 			// When coordinates are fractional, GS appears to draw to the right/bottom (effectively
 			// taking the ceiling), not to the top/left (taking the floor).
@@ -3870,11 +3883,24 @@ GSState::TextureMinMaxResult GSState::GetTextureMinMax(GIFRegTEX0 TEX0, GIFRegCL
 
 				const GSVertex* vert_first = &m_vertex.buff[m_index.buff[0]];
 				const GSVertex* vert_second = &m_vertex.buff[m_index.buff[1]];
+				const GSVertex* vert_third = &m_vertex.buff[m_index.buff[2]];
 
 				GSVector4 new_st = st;
+				bool u_forward_check = false;
+				bool x_forward_check = false;
+				if (m_vt.m_primclass == GS_TRIANGLE_CLASS)
+				{
+					u_forward_check = PRIM->FST ? ((vert_first->U < vert_second->U) || (vert_first->U < vert_third->U)) : (((vert_first->ST.S / vert_first->RGBAQ.Q) < (vert_second->ST.S / vert_second->RGBAQ.Q)) || ((vert_first->ST.S / vert_first->RGBAQ.Q) < (vert_third->ST.S / vert_third->RGBAQ.Q)));
+					x_forward_check = (vert_first->XYZ.X < vert_second->XYZ.X) || (vert_first->XYZ.X < vert_third->XYZ.X);
+				}
+				else
+				{
+					u_forward_check = PRIM->FST ? (vert_first->U < vert_second->U) : ((vert_first->ST.T / vert_first->RGBAQ.Q) < (vert_second->ST.T / vert_first->RGBAQ.Q));
+					x_forward_check = vert_first->XYZ.Y < vert_second->XYZ.Y;
+				}
 				// Check if the UV coords are going in a different direction to the verts, if they match direction, no need to swap
-				const bool u_forward = vert_first->U < vert_second->U;
-				const bool x_forward = vert_first->XYZ.X < vert_second->XYZ.X;
+				const bool u_forward = u_forward_check;
+				const bool x_forward = x_forward_check;
 				const bool swap_x = u_forward != x_forward;
 
 				if (int_rc.left < scissored_rc.left)
@@ -3897,9 +3923,20 @@ GSState::TextureMinMaxResult GSState::GetTextureMinMax(GIFRegTEX0 TEX0, GIFRegCL
 					st.x = new_st.x;
 					st.z = new_st.z;
 				}
-
-				const bool v_forward = vert_first->V < vert_second->V;
-				const bool y_forward = vert_first->XYZ.Y < vert_second->XYZ.Y;
+				bool v_forward_check = false;
+				bool y_forward_check = false;
+				if (m_vt.m_primclass == GS_TRIANGLE_CLASS)
+				{
+					v_forward_check = PRIM->FST ? ((vert_first->V < vert_second->V) || (vert_first->V < vert_third->V)) : (((vert_first->ST.T / vert_first->RGBAQ.Q) < (vert_second->ST.T / vert_second->RGBAQ.Q)) || ((vert_first->ST.T / vert_first->RGBAQ.Q) < (vert_third->ST.T / vert_third->RGBAQ.Q)));
+					y_forward_check = (vert_first->XYZ.Y < vert_second->XYZ.Y) || (vert_first->XYZ.Y < vert_third->XYZ.Y);
+				}
+				else
+				{
+					v_forward_check = PRIM->FST ? (vert_first->V < vert_second->V) : ((vert_first->ST.T / vert_first->RGBAQ.Q) < (vert_second->ST.T / vert_first->RGBAQ.Q));
+					y_forward_check = vert_first->XYZ.Y < vert_second->XYZ.Y;
+				}
+				const bool v_forward = v_forward_check;
+				const bool y_forward = y_forward_check;
 				const bool swap_y = v_forward != y_forward;
 
 				if (int_rc.top < scissored_rc.top)

pcsx2/GS/GSState.h

@@ -224,6 +224,8 @@ class GSState : public GSAlignedClass<32>
 	bool m_texflush_flag = false;
 	bool m_isPackedUV_HackFlag = false;
 	bool m_channel_shuffle = false;
+	bool m_in_target_draw = false;
+	u32 m_target_offset = 0;
 	u8 m_scanmask_used = 0;
 	u32 m_dirty_gs_regs = 0;
 	int m_backed_up_ctx = 0;

pcsx2/GS/GSVector4i.h

@@ -1599,6 +1599,11 @@ class alignas(16) GSVector4i
 		return loadh(&v);
 	}
 
+	__forceinline static GSVector4i loadl(const GSVector2i& v)
+	{
+		return loadl(&v);
+	}
+
 	__forceinline static GSVector4i load(const void* pl, const void* ph)
 	{
 		return loadh(ph, loadl(pl));

pcsx2/GS/Renderers/HW/GSHwHack.cpp

@@ -194,7 +194,7 @@ bool GSHwHack::GSC_Tekken5(GSRendererHW& r, int& skip)
 			return true;
 		}
 
-		if (!s_nativeres && r.PRIM->PRIM == GS_SPRITE && RTME && RTEX0.TFX == 1 && RFPSM == RTPSM && RTPSM == PSMCT32 && RFBMSK == 0xFF000000 && r.m_index.tail > 2)
+		if (!s_nativeres && r.PRIM->PRIM == GS_SPRITE && RTME && RTEX0.TFX == 1 && !r.PRIM->ABE && RFPSM == RTPSM && RTPSM == PSMCT32 && RFBMSK == 0xFF000000 && r.m_index.tail > 2)
 		{
 			// Don't enable hack on native res.
 			// Fixes ghosting/blur effect and white lines appearing in stages: Moonfit Wilderness, Acid Rain - caused by upscaling.
@@ -204,12 +204,6 @@ bool GSHwHack::GSC_Tekken5(GSRendererHW& r, int& skip)
 			const GSVector4i read_size(r.m_vt.m_min.t.x, r.m_vt.m_min.t.y, r.m_vt.m_max.t.x + 0.5f, r.m_vt.m_max.t.y + 0.5f);
 			r.ReplaceVerticesWithSprite(draw_size, read_size, GSVector2i(read_size.width(), read_size.height()), draw_size);
 		}
-		else if (RZTST == 1 && RTME && (RFBP == 0x02bc0 || RFBP == 0x02be0 || RFBP == 0x02d00 || RFBP == 0x03480 || RFBP == 0x034a0) && RFPSM == RTPSM && RTBP0 == 0x00000 && RTPSM == PSMCT32)
-		{
-			// The moving display effect(flames) is not emulated properly in the entire screen so let's remove the effect in the stage: Burning Temple. Related to half screen bottom issue.
-			// Fixes black lines in the stage: Burning Temple - caused by upscaling. Note the black lines can also be fixed with Merge Sprite hack.
-			skip = 2;
-		}
 	}
 
 	return true;
@@ -1047,7 +1041,7 @@ bool GSHwHack::OI_SonicUnleashed(GSRendererHW& r, GSTexture* rt, GSTexture* ds,
 	// compute shadow in RG,
 	// save result in alpha with a TS,
 	// Restore RG channel that we previously copied to render shadows.
-
+	// Important note: The game downsizes the target to half height, then later expands it back up to full size, that's why PCSX2 doesn't like it, we don't support that behaviour.
 	const GIFRegTEX0& Texture = RTEX0;
 
 	GIFRegTEX0 Frame = {};
@@ -1058,9 +1052,9 @@ bool GSHwHack::OI_SonicUnleashed(GSRendererHW& r, GSTexture* rt, GSTexture* ds,
 	if ((!rt) || (!RPRIM->TME) || (GSLocalMemory::m_psm[Texture.PSM].bpp != 16) || (GSLocalMemory::m_psm[Frame.PSM].bpp != 16) || (Texture.TBP0 == Frame.TBP0) || (Frame.TBW != 16 && Texture.TBW != 16))
 		return true;
 
-	GL_INS("OI_SonicUnleashed replace draw by a copy");
+	GL_INS("OI_SonicUnleashed replace draw by a copy draw %d", r.s_n);
 
-	GSTextureCache::Target* src = g_texture_cache->LookupTarget(Texture, GSVector2i(1, 1), r.GetTextureScaleFactor(), GSTextureCache::RenderTarget);
+	GSTextureCache::Target* src = g_texture_cache->LookupTarget(Texture, GSVector2i(1, 1), r.GetTextureScaleFactor(), GSTextureCache::RenderTarget, true, 0, false, false, true, true, GSVector4i::zero(), true);
 
 	if (!src)
 		return true;
@@ -1086,6 +1080,7 @@ bool GSHwHack::OI_SonicUnleashed(GSRendererHW& r, GSTexture* rt, GSTexture* ds,
 	const GSVector2i copy_size(std::min(rt_size.x, src_size.x), std::min(rt_size.y, src_size.y));
 
 	const GSVector4 sRect(0.0f, 0.0f, static_cast<float>(copy_size.x) / static_cast<float>(src_size.x), static_cast<float>(copy_size.y) / static_cast<float>(src_size.y));
+	// This is kind of a bodge because the game confuses everything since the source is really 16bit and it assumes it's really drawing 16bit on the copy back, resizing the target.
 	const GSVector4 dRect(0, 0, copy_size.x, copy_size.y);
 
 	g_gs_device->StretchRect(src->m_texture, sRect, rt, dRect, true, true, true, false);