ChdFileReader: implement precaching of CHD files within PCSX2 instead of modifying libchdr

3rdparty/libchdr/include/libchdr/chd.h

@@ -290,8 +290,7 @@ enum _chd_error
 	CHDERR_INVALID_STATE,
 	CHDERR_OPERATION_PENDING,
 	CHDERR_NO_ASYNC_OPERATION,
-	CHDERR_UNSUPPORTED_FORMAT,
-	CHDERR_CANCELLED,
+	CHDERR_UNSUPPORTED_FORMAT
 };
 typedef enum _chd_error chd_error;
 
@@ -383,17 +382,13 @@ CHD_EXPORT chd_error chd_open(const char *filename, int mode, chd_file *parent,
 
 /* precache underlying file */
 CHD_EXPORT chd_error chd_precache(chd_file *chd);
-CHD_EXPORT chd_error chd_precache_progress(chd_file* chd, bool(*progress)(size_t pos, size_t total, void* param), void* param);
 
 /* close a CHD file */
 CHD_EXPORT void chd_close(chd_file *chd);
 
 /* return the associated core_file */
 CHD_EXPORT core_file *chd_core_file(chd_file *chd);
 
-/* return the overall size of a CHD, and any of its parents */
-CHD_EXPORT UINT64 chd_get_compressed_size(chd_file* chd);
-
 /* return an error string for the given CHD error */
 CHD_EXPORT const char *chd_error_string(chd_error err);
 

3rdparty/libchdr/src/libchdr_chd.c

@@ -1953,56 +1953,28 @@ CHD_EXPORT chd_error chd_open_core_file(core_file *file, int mode, chd_file *par
 
 CHD_EXPORT chd_error chd_precache(chd_file* chd)
 {
-  return chd_precache_progress(chd, NULL, NULL);
-}
-
-CHD_EXPORT chd_error chd_precache_progress(chd_file* chd, bool(*progress)(size_t pos, size_t total, void* param), void* param)
-{
-#define PRECACHE_CHUNK_SIZE 16 * 1024 * 1024
+	INT64 count;
+	UINT64 size;
 
-  if (chd->file_cache == NULL)
-  {
-    const UINT64 size = core_fsize(chd->file);
-    if ((INT64)size <= 0)
-      return CHDERR_INVALID_DATA;
-
-		if (size > SIZE_MAX)
+	if (chd->file_cache == NULL)
+	{
+		size = core_fsize(chd->file);
+		if ((INT64)size <= 0)
+			return CHDERR_INVALID_DATA;
+		chd->file_cache = malloc(size);
+		if (chd->file_cache == NULL)
 			return CHDERR_OUT_OF_MEMORY;
-
-    chd->file_cache = malloc(size);
-    if (chd->file_cache == NULL)
-      return CHDERR_OUT_OF_MEMORY;
-    core_fseek(chd->file, 0, SEEK_SET);
-
-		UINT64 done = 0;
-		while (done < size)
+		core_fseek(chd->file, 0, SEEK_SET);
+		count = core_fread(chd->file, chd->file_cache, size);
+		if (count != size)
 		{
-			UINT64 req_count = size - done;
-			if (req_count > PRECACHE_CHUNK_SIZE)
-				req_count = PRECACHE_CHUNK_SIZE;
-
-			size_t count = core_fread(chd->file, chd->file_cache + (size_t)done, (size_t)req_count);
-			if (count != (size_t)req_count)
-			{
-				free(chd->file_cache);
-				chd->file_cache = NULL;
-				return CHDERR_READ_ERROR;
-			}
-
-			done += req_count;
-			if (progress != NULL)
-			{
-				if (!progress(done, size, param))
-				{
-					free(chd->file_cache);
-					chd->file_cache = NULL;
-					return CHDERR_CANCELLED;
-				}
-			}
+			free(chd->file_cache);
+			chd->file_cache = NULL;
+			return CHDERR_READ_ERROR;
 		}
-  }
+	}
 
-  return CHDERR_NONE;
+	return CHDERR_NONE;
 }
 
 /*-------------------------------------------------
@@ -2169,14 +2141,6 @@ CHD_EXPORT core_file *chd_core_file(chd_file *chd)
 	return chd->file;
 }
 
-CHD_EXPORT UINT64 chd_get_compressed_size(chd_file *chd)
-{
-	UINT64 size = chd->file->fsize(chd->file);
-	if (chd->parent)
-		size += chd_get_compressed_size(chd->parent);
-	return size;
-}
-
 /*-------------------------------------------------
     chd_error_string - return an error string for
     the given CHD error

common/FileSystem.cpp

@@ -1230,6 +1230,14 @@ size_t FileSystem::ReadFileWithProgress(std::FILE* fp, void* dst, size_t length,
 {
 	progress->SetProgressRange(100);
 
+	return FileSystem::ReadFileWithPartialProgress(fp, dst, length, progress, 0, 100, error, chunk_size);
+}
+
+size_t FileSystem::ReadFileWithPartialProgress(std::FILE* fp, void* dst, size_t length,
+	ProgressCallback* progress, int startPercent, int endPercent, Error* error, size_t chunk_size)
+{
+	const int deltaPercent = endPercent - startPercent;
+
 	size_t done = 0;
 	while (done < length)
 	{
@@ -1243,7 +1251,7 @@ size_t FileSystem::ReadFileWithProgress(std::FILE* fp, void* dst, size_t length,
 			break;
 		}
 
-		progress->SetProgressValue((done * 100) / length);
+		progress->SetProgressValue(startPercent + (done * deltaPercent) / length);
 		done += read_size;
 	}
 

common/FileSystem.h

@@ -144,6 +144,8 @@ namespace FileSystem
 	bool WriteStringToFile(const char* filename, const std::string_view sv);
 	size_t ReadFileWithProgress(std::FILE* fp, void* dst, size_t length, ProgressCallback* progress,
 		Error* error = nullptr, size_t chunk_size = 16 * 1024 * 1024);
+	size_t ReadFileWithPartialProgress(std::FILE* fp, void* dst, size_t length, ProgressCallback* progress,
+		int startPercent, int endPercent, Error* error = nullptr, size_t chunk_size = 16 * 1024 * 1024);
 
 	/// creates a directory in the local filesystem
 	/// if the directory already exists, the return value will be true.

pcsx2/CDVD/ChdFileReader.cpp

@@ -21,6 +21,7 @@ static std::vector<std::pair<std::string, chd_header>> s_chd_hash_cache; // <fil
 static std::recursive_mutex s_chd_hash_cache_mutex;
 
 // Provides an implementation of core_file which allows us to control if the underlying FILE handle is freed.
+// Additionally, this class allows greater control and feedback while precaching CHD files.
 // The lifetime of ChdCoreFileWrapper will be equal to that of the relevant chd_file,
 // ChdCoreFileWrapper will also get destroyed if chd_open_core_file fails.
 class ChdCoreFileWrapper
@@ -31,10 +32,15 @@ class ChdCoreFileWrapper
 	core_file m_core;
 	std::FILE* m_file;
 	bool m_free_file = false;
+	ChdCoreFileWrapper* m_parent = nullptr;
+	std::unique_ptr<u8[]> m_file_cache;
+	s64 m_file_cache_size;
+	s64 m_file_cache_pos;
 
 public:
-	ChdCoreFileWrapper(std::FILE* file)
+	ChdCoreFileWrapper(std::FILE* file, ChdCoreFileWrapper* parent)
 		: m_file{file}
+		, m_parent{parent}
 	{
 		m_core.argp = this;
 		m_core.fsize = FSize;
@@ -45,7 +51,7 @@ class ChdCoreFileWrapper
 
 	~ChdCoreFileWrapper()
 	{
-		if (m_free_file)
+		if (m_free_file && m_file)
 			std::fclose(m_file);
 	}
 
@@ -64,15 +70,100 @@ class ChdCoreFileWrapper
 		m_free_file = isOwner;
 	}
 
+	s64 GetPrecacheSize()
+	{
+		const s64 size = static_cast<size_t>(FileSystem::FSize64(m_file));
+		if (m_parent != nullptr)
+			return m_parent->GetPrecacheSize() + size;
+		else
+			return size;
+	}
+
+	bool Precache(ProgressCallback* progress, Error* error)
+	{
+		progress->SetProgressRange(100);
+
+		const s64 size = GetPrecacheSize();
+		return PrecacheInternal(progress, error, 0, size);
+	}
+
 private:
+	bool PrecacheInternal(ProgressCallback* progress, Error* error, s64 startSize, s64 finalSize)
+	{
+		m_file_cache_size = FileSystem::FSize64(m_file);
+		if (m_file_cache_size <= 0)
+		{
+			Error::SetStringView(error, "Failed to determine file size.");
+			return false;
+		}
+
+		// Copy the current file position.
+		m_file_cache_pos = FileSystem::FTell64(m_file);
+		if (m_file_cache_pos <= 0)
+		{
+			Error::SetStringView(error, "Failed to determine file position.");
+			return false;
+		}
+
+		m_file_cache = std::make_unique_for_overwrite<u8[]>(m_file_cache_size);
+		if (FileSystem::FSeek64(m_file, 0, SEEK_SET) != 0 ||
+			FileSystem::ReadFileWithPartialProgress(
+				m_file, m_file_cache.get(), m_file_cache_size, progress,
+				(startSize * 100) / finalSize,
+				((startSize + m_file_cache_size) * 100) / finalSize,
+				error) != static_cast<size_t>(m_file_cache_size))
+		{
+			m_file_cache.reset();
+			// Precache failed, continue using file
+			// Restore file position incase it's used for subsequent reads
+			FileSystem::FSeek64(m_file, m_file_cache_pos, SEEK_SET);
+			Error::SetStringView(error, "Failed to read part of the file.");
+			return false;
+		}
+
+		startSize += m_file_cache_size;
+
+		if (m_parent)
+		{
+			if (!m_parent->PrecacheInternal(progress, error, startSize, finalSize))
+			{
+				// Precache failed, continue using file
+				// Restore file position incase it's used for subsequent reads
+				FileSystem::FSeek64(m_file, m_file_cache_pos, SEEK_SET);
+				m_file_cache.reset();
+				return false;
+			}
+		}
+
+		if (m_free_file)
+			std::fclose(m_file);
+		m_file = nullptr;
+
+		return true;
+	}
+
 	static u64 FSize(core_file* file)
 	{
-		return static_cast<u64>(FileSystem::FSize64(FromCoreFile(file)->m_file));
+		ChdCoreFileWrapper* fileWrapper = FromCoreFile(file);
+		if (fileWrapper->m_file_cache)
+			return fileWrapper->m_file_cache_size;
+		else
+			return static_cast<u64>(FileSystem::FSize64(fileWrapper->m_file));
 	}
 
 	static size_t FRead(void* buffer, size_t elmSize, size_t elmCount, core_file* file)
 	{
-		return std::fread(buffer, elmSize, elmCount, FromCoreFile(file)->m_file);
+		ChdCoreFileWrapper* fileWrapper = FromCoreFile(file);
+		if (fileWrapper->m_file_cache)
+		{
+			// While currently libchdr only uses an elmCount of 1, we can't guarantee that will always be the case.
+			elmCount = std::min<size_t>(elmCount, std::max<s64>(fileWrapper->m_file_cache_size - fileWrapper->m_file_cache_pos, 0) / elmSize);
+			const size_t size = elmSize * elmCount;
+			std::memcpy(buffer, &fileWrapper->m_file_cache[fileWrapper->m_file_cache_pos], size);
+			return elmCount;
+		}
+		else
+			return std::fread(buffer, elmSize, elmCount, fileWrapper->m_file);
 	}
 
 	static int FClose(core_file* file)
@@ -84,7 +175,28 @@ class ChdCoreFileWrapper
 
 	static int FSeek(core_file* file, int64_t offset, int whence)
 	{
-		return FileSystem::FSeek64(FromCoreFile(file)->m_file, offset, whence);
+		ChdCoreFileWrapper* fileWrapper = FromCoreFile(file);
+		if (fileWrapper->m_file_cache)
+		{
+			switch (whence)
+			{
+				case SEEK_SET:
+					fileWrapper->m_file_cache_pos = offset;
+					break;
+				case SEEK_CUR:
+					fileWrapper->m_file_cache_pos += offset;
+					break;
+				case SEEK_END:
+					fileWrapper->m_file_cache_pos = fileWrapper->m_file_cache_size + offset;
+					break;
+				default:
+					return -1;
+			}
+
+			return 0;
+		}
+		else
+			return FileSystem::FSeek64(fileWrapper->m_file, offset, whence);
 	}
 };
 
@@ -98,7 +210,7 @@ ChdFileReader::~ChdFileReader()
 static chd_file* OpenCHD(const std::string& filename, FileSystem::ManagedCFilePtr fp, Error* error, u32 recursion_level)
 {
 	chd_file* chd;
-	ChdCoreFileWrapper* core_wrapper = new ChdCoreFileWrapper(fp.get());
+	ChdCoreFileWrapper* core_wrapper = new ChdCoreFileWrapper(fp.get(), nullptr);
 	// libchdr will take ownership of core_wrapper, and will close/free it on failure.
 	chd_error err = chd_open_core_file(core_wrapper->GetCoreFile(), CHD_OPEN_READ, nullptr, &chd);
 	if (err == CHDERR_NONE)
@@ -212,7 +324,7 @@ static chd_file* OpenCHD(const std::string& filename, FileSystem::ManagedCFilePt
 	}
 
 	// Our last core file wrapper got freed, so make a new one.
-	core_wrapper = new ChdCoreFileWrapper(fp.get());
+	core_wrapper = new ChdCoreFileWrapper(fp.get(), ChdCoreFileWrapper::FromCoreFile(chd_core_file(parent_chd)));
 	// Now try re-opening with the parent.
 	err = chd_open_core_file(core_wrapper->GetCoreFile(), CHD_OPEN_READ, parent_chd, &chd);
 	if (err != CHDERR_NONE)
@@ -266,28 +378,11 @@ bool ChdFileReader::Open2(std::string filename, Error* error)
 
 bool ChdFileReader::Precache2(ProgressCallback* progress, Error* error)
 {
-	if (!CheckAvailableMemoryForPrecaching(chd_get_compressed_size(ChdFile), error))
-		return false;
-
-	progress->SetProgressRange(100);
-
-	const auto callback = [](size_t pos, size_t total, void* param) -> bool {
-		ProgressCallback* progress = static_cast<ProgressCallback*>(param);
-		const u32 percent = static_cast<u32>((pos * 100) / total);
-		progress->SetProgressValue(std::min<u32>(percent, 100));
-		return !progress->IsCancelled();
-	};
-
-	const chd_error cerror = chd_precache_progress(ChdFile, callback, progress);
-	if (cerror != CHDERR_NONE)
-	{
-		if (cerror != CHDERR_CANCELLED)
-			Error::SetStringView(error, "Failed to read part of the file.");
-
+	ChdCoreFileWrapper* fileWrapper = ChdCoreFileWrapper::FromCoreFile(chd_core_file(ChdFile));
+	if (!CheckAvailableMemoryForPrecaching(fileWrapper->GetPrecacheSize(), error))
 		return false;
-	}
 
-	return true;
+	return fileWrapper->Precache(progress, error);
 }
 
 ThreadedFileReader::Chunk ChdFileReader::ChunkForOffset(u64 offset)