From 0c1ba069f823d62b3d7dd20e5813e0eca0865e92 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Fri, 29 Mar 2019 00:44:27 +0500 Subject: [PATCH] implement disc validation --- IrdLibraryClient/IrdFormat/IsoHeaderParser.cs | 3 +- Ps3DiscDumper/Decrypter.cs | 25 ++++- Ps3DiscDumper/DiscInfo/DiscInfo.cs | 13 +-- Ps3DiscDumper/DiscInfo/DiscInfoConverter.cs | 50 ++++++++++ Ps3DiscDumper/DiscInfo/FileInfo.cs | 11 +++ Ps3DiscDumper/Dumper.cs | 95 +++++++++++-------- UI.WinForms.Msil/Forms/MainForm.Designer.cs | 2 +- UI.WinForms.Msil/Forms/MainForm.cs | 4 +- 8 files changed, 145 insertions(+), 58 deletions(-) create mode 100644 Ps3DiscDumper/DiscInfo/DiscInfoConverter.cs create mode 100644 Ps3DiscDumper/DiscInfo/FileInfo.cs diff --git a/IrdLibraryClient/IrdFormat/IsoHeaderParser.cs b/IrdLibraryClient/IrdFormat/IsoHeaderParser.cs index 45119ba..132f199 100644 --- a/IrdLibraryClient/IrdFormat/IsoHeaderParser.cs +++ b/IrdLibraryClient/IrdFormat/IsoHeaderParser.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; using System.Linq; using DiscUtils.Iso9660; diff --git a/Ps3DiscDumper/Decrypter.cs b/Ps3DiscDumper/Decrypter.cs index e1bcc91..aa3339a 100644 --- a/Ps3DiscDumper/Decrypter.cs +++ b/Ps3DiscDumper/Decrypter.cs @@ -18,8 +18,11 @@ public class Decrypter : Stream, IDisposable private byte[] decryptionKey; private readonly int sectorSize; private readonly MD5 md5; + private readonly SHA1 sha1; + private readonly SHA256 sha256; private readonly Aes aes; - private byte[] bufferedSector, tmpSector, hash = null; + private byte[] bufferedSector, tmpSector; + private Dictionary hashes = new Dictionary(3); private readonly List<(int start, int end)> unprotectedSectorRanges; public static byte[] DecryptDiscKey(byte[] data1) @@ -92,6 +95,8 @@ public Decrypter(Stream fileStream, Stream discStream, byte[] decryptionKey, lon this.discStream = discStream; this.decryptionKey = decryptionKey; md5 = MD5.Create(); + sha1 = SHA1.Create(); + sha256 = SHA256.Create(); aes = Aes.Create(); aes.Mode = CipherMode.CBC; aes.Padding = PaddingMode.None; @@ -113,6 +118,8 @@ public override int Read( byte[] buffer, int offset, int count) { var len = (int)Math.Min(Math.Min(count, sectorSize - positionInSector), inputStream.Position - Position); md5.TransformBlock(bufferedSector, (int)positionInSector, len, buffer, offset); + sha1.TransformBlock(bufferedSector, (int)positionInSector, len, buffer, offset); + sha256.TransformBlock(bufferedSector, (int)positionInSector, len, buffer, offset); offset += len; count -= len; resultCount += len; @@ -151,6 +158,8 @@ public override int Read( byte[] buffer, int offset, int count) if (count >= readCount) { md5.TransformBlock(decryptedSector, 0, readCount, buffer, offset); + sha1.TransformBlock(decryptedSector, 0, readCount, buffer, offset); + sha256.TransformBlock(decryptedSector, 0, readCount, buffer, offset); offset += readCount; count -= readCount; resultCount += readCount; @@ -161,6 +170,8 @@ public override int Read( byte[] buffer, int offset, int count) { Buffer.BlockCopy(decryptedSector, 0, bufferedSector, 0, sectorSize); md5.TransformBlock(decryptedSector, 0, count, buffer, offset); + sha1.TransformBlock(decryptedSector, 0, count, buffer, offset); + sha256.TransformBlock(decryptedSector, 0, count, buffer, offset); offset += count; count = 0; resultCount += count; @@ -170,14 +181,18 @@ public override int Read( byte[] buffer, int offset, int count) return resultCount; } - public byte[] GetMd5() + public Dictionary GetHashes() { - if (hash == null) + if (hashes.Count == 0) { md5.TransformFinalBlock(tmpSector, 0, 0); - hash = md5.Hash; + sha1.TransformFinalBlock(tmpSector, 0, 0); + sha256.TransformFinalBlock(tmpSector, 0, 0); + hashes["MD5"] = md5.Hash.ToHexString(); + hashes["SHA1"] = sha1.Hash.ToHexString(); + hashes["SHA256"] = sha256.Hash.ToHexString(); } - return hash; + return hashes; } private bool IsEncrypted(long sector) diff --git a/Ps3DiscDumper/DiscInfo/DiscInfo.cs b/Ps3DiscDumper/DiscInfo/DiscInfo.cs index 70ca195..5ffd285 100644 --- a/Ps3DiscDumper/DiscInfo/DiscInfo.cs +++ b/Ps3DiscDumper/DiscInfo/DiscInfo.cs @@ -2,22 +2,13 @@ namespace Ps3DiscDumper.DiscInfo { - using Hashes = Dictionary; // hash type - hash value - public class DiscInfo { public string ProductCode; // BLUS12345 public string DiscVersion; // VERSION field from PARAM.SFO - public byte[] DiscKeyRawData; // IRD - public byte[] DiscKey; // Redump, and what is actually used for decryption + public string DiscKeyRawData; // IRD + public string DiscKey; // Redump, and what is actually used for decryption public FileInfo DiscImage; public Dictionary Files; } - - public class FileInfo - { - public long Offset; - public long Size; - public Hashes Hashes; - } } diff --git a/Ps3DiscDumper/DiscInfo/DiscInfoConverter.cs b/Ps3DiscDumper/DiscInfo/DiscInfoConverter.cs new file mode 100644 index 0000000..71be3e2 --- /dev/null +++ b/Ps3DiscDumper/DiscInfo/DiscInfoConverter.cs @@ -0,0 +1,50 @@ +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using DiscUtils.Iso9660; +using IrdLibraryClient.IrdFormat; +using Ps3DiscDumper.Utils; + +namespace Ps3DiscDumper.DiscInfo +{ + public static class DiscInfoConverter + { + public static DiscInfo ToDiscInfo(this Ird ird) + { + List fsInfo; + var sectorSize = 2048L; + using (var stream = new MemoryStream()) + { + using (var headerStream = new MemoryStream(ird.Header)) + using (var gzipStream = new GZipStream(headerStream, CompressionMode.Decompress)) + { + gzipStream.CopyTo(stream); + } + stream.Seek(0, SeekOrigin.Begin); + var reader = new CDReader(stream, true, true); + fsInfo = reader.GetFilesystemStructure(); + sectorSize = reader.ClusterSize; + } + var checksums = ird.Files.ToDictionary(f => f.Offset, f => f.Md5Checksum.ToHexString()); + return new DiscInfo + { + ProductCode = ird.ProductCode, + DiscVersion = ird.GameVersion, + DiscKeyRawData = ird.Data1.ToHexString(), + DiscKey = Decrypter.DecryptDiscKey(ird.Data1).ToHexString(), + Files = fsInfo.ToDictionary( + f => f.Filename, + f => new FileInfo + { + Offset = f.StartSector * sectorSize, + Size = f.Length, + Hashes = new Dictionary + { + ["MD5"] = checksums[f.StartSector], + } + }) + }; + } + } +} diff --git a/Ps3DiscDumper/DiscInfo/FileInfo.cs b/Ps3DiscDumper/DiscInfo/FileInfo.cs new file mode 100644 index 0000000..b7ae222 --- /dev/null +++ b/Ps3DiscDumper/DiscInfo/FileInfo.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace Ps3DiscDumper.DiscInfo +{ + public class FileInfo + { + public long Offset; + public long Size; + public Dictionary Hashes; + } +} \ No newline at end of file diff --git a/Ps3DiscDumper/Dumper.cs b/Ps3DiscDumper/Dumper.cs index 3357b30..ef12415 100644 --- a/Ps3DiscDumper/Dumper.cs +++ b/Ps3DiscDumper/Dumper.cs @@ -12,10 +12,12 @@ using IrdLibraryClient; using IrdLibraryClient.IrdFormat; using IrdLibraryClient.POCOs; +using Ps3DiscDumper.DiscInfo; using Ps3DiscDumper.DiscKeyProviders; using Ps3DiscDumper.Sfb; using Ps3DiscDumper.Sfo; using Ps3DiscDumper.Utils; +using FileInfo = System.IO.FileInfo; namespace Ps3DiscDumper { @@ -35,6 +37,7 @@ public class Dumper: IDisposable public ParamSfo ParamSfo { get; private set; } public string ProductCode { get; private set; } + public string DiscVersion { get; private set; } public string Title { get; private set; } public string OutputDir { get; private set; } public char Drive { get; set; } @@ -135,28 +138,11 @@ private void CheckParamSfo(ParamSfo sfo) if (titleParts.Length > 1) Title = string.Join(" ", titleParts); ProductCode = sfo.Items.FirstOrDefault(i => i.Key == "TITLE_ID")?.StringValue?.Trim(' ', '\0'); + DiscVersion = sfo.Items.FirstOrDefault(i => i.Key == "VERSION")?.StringValue?.Trim(); Log.Info($"Game title: {Title}"); } -/* - public bool IsFilenameSetMatch(Ird ird) - { - var expectedSet = new HashSet(DiscFilenames, StringComparer.InvariantCultureIgnoreCase); - var irdSet = new HashSet(StringComparer.InvariantCultureIgnoreCase); - var fileInfo = ird.GetFilesystemStructure(); - foreach (var file in fileInfo) - { - if (Cts.IsCancellationRequested) - return false; - - var convertedFilename = Path.DirectorySeparatorChar == '\\' ? file.Filename : file.Filename.Replace('\\', Path.DirectorySeparatorChar); - irdSet.Add(convertedFilename); - } - return expectedSet.SetEquals(irdSet); - } -*/ - public void DetectDisc(Func outputDirFormatter = null) { outputDirFormatter = outputDirFormatter ?? (d => $"[{d.ProductCode}] {d.Title}"); @@ -343,7 +329,7 @@ public async Task FindDiscKeyAsync(string discKeyCachePath) lock (AllKnownDiscKeys) AllKnownDiscKeys.TryGetValue(discKey, out allMatchingKeys); - DiscKeyFilename = Path.GetFileName(allMatchingKeys.First().FullPath); + DiscKeyFilename = Path.GetFileName(allMatchingKeys?.First().FullPath); } public async Task DumpAsync(string output) @@ -358,7 +344,9 @@ public async Task DumpAsync(string output) else dumpPath = parent; } - filesystemStructure = filesystemStructure ?? discReader.GetFilesystemStructure(); + var getFsTask = Task.Run(() => filesystemStructure ?? discReader.GetFilesystemStructure()); + var validators = await Task.Run(GetValidationInfo).ConfigureAwait(false); + filesystemStructure = await getFsTask.ConfigureAwait(false); if (!string.IsNullOrEmpty(dumpPath)) { var root = Path.GetPathRoot(Path.GetFullPath(output)); @@ -385,6 +373,7 @@ public async Task DumpAsync(string output) var decryptionKey = allMatchingKeys.First().DecryptedKey; var sectorSize = (int)discReader.ClusterSize; var unprotectedRegions = driveStream.GetUnprotectedRegions(); + ValidationStatus = true; foreach (var file in filesystemStructure) { @@ -409,9 +398,12 @@ public async Task DumpAsync(string output) Directory.CreateDirectory(fileDir); var error = false; - //var expectedMd5 = Ird.Files.First(f => f.Offset == file.StartSector).Md5Checksum.ToHexString(); - //var lastMd5 = expectedMd5; - string lastMd5 = null; + var expectedHashes = ( + from v in validators + where v.Files.ContainsKey(file.Filename) + select v.Files[file.Filename].Hashes + ).ToList(); + var lastHash = ""; do { try @@ -422,26 +414,32 @@ public async Task DumpAsync(string output) { await Decrypter.CopyToAsync(outputStream, 8 * 1024 * 1024, Cts.Token).ConfigureAwait(false); outputStream.Flush(); - var resultMd5 = Decrypter.GetMd5().ToHexString(); + var resultHashes = Decrypter.GetHashes(); + var resultMd5 = resultHashes["MD5"]; if (Decrypter.WasEncrypted && Decrypter.WasUnprotected) Log.Debug("Partially decrypted"); else if (Decrypter.WasEncrypted) Log.Debug("Decrypted"); -/* - if (expectedMd5 != resultMd5) + + if (!expectedHashes.Any()) + { + if (ValidationStatus == true) + ValidationStatus = null; + } + else if (!IsMatch(resultHashes, expectedHashes)) + { + error = true; + var msg = "Unexpected hash: " + resultMd5; + if (resultMd5 == lastHash || Decrypter.LastBlockCorrupted) { - error = true; - var msg = $"Expected {expectedMd5}, but was {resultMd5}"; - if (lastMd5 == resultMd5 || Decrypter.LastBlockCorrupted) - { - Log.Error(msg); - BrokenFiles.Add((file.Filename, "corrupted")); - break; - } - Log.Warn(msg + ", retrying"); + Log.Error(msg); + BrokenFiles.Add((file.Filename, "corrupted")); + break; } -*/ - lastMd5 = resultMd5; + Log.Warn(msg + ", retrying"); + } + + lastHash = resultMd5; } } catch (Exception e) @@ -454,6 +452,20 @@ public async Task DumpAsync(string output) Log.Info("Completed"); } + private List GetValidationInfo() + { + var discInfoList = new List(); + foreach (var discKeyInfo in allMatchingKeys.Where(ki => ki.KeyType == KeyType.Ird)) + { + var ird = IrdParser.Parse(File.ReadAllBytes(discKeyInfo.FullPath)); + if (!DiscVersion.Equals(ird.GameVersion)) + continue; + + discInfoList.Add(ird.ToDiscInfo()); + } + return discInfoList; + } + private bool IsValidDiscKey(string discKeyId) { HashSet keys; @@ -476,6 +488,15 @@ public bool IsValidDiscKey(byte[] discKey) } + private static bool IsMatch(Dictionary hashes, List> expectedHashes) + { + foreach (var eh in expectedHashes) + foreach (var h in hashes) + if (eh.TryGetValue(h.Key, out var expectedHash) && expectedHash == h.Value) + return true; + return false; + } + public void Dispose() { driveStream?.Dispose(); diff --git a/UI.WinForms.Msil/Forms/MainForm.Designer.cs b/UI.WinForms.Msil/Forms/MainForm.Designer.cs index b5f3c24..eee41e1 100644 --- a/UI.WinForms.Msil/Forms/MainForm.Designer.cs +++ b/UI.WinForms.Msil/Forms/MainForm.Designer.cs @@ -338,7 +338,7 @@ private void InitializeComponent() this.MaximizeBox = false; this.Name = "MainForm"; this.StartPosition = System.Windows.Forms.FormStartPosition.CenterScreen; - this.Text = "PS3 Disc Dumper v3.0 beta 1"; + this.Text = "PS3 Disc Dumper v3.0"; this.FormClosing += new System.Windows.Forms.FormClosingEventHandler(this.MainForm_FormClosing); this.Load += new System.EventHandler(this.MainForm_Load); this.Shown += new System.EventHandler(this.MainForm_Shown); diff --git a/UI.WinForms.Msil/Forms/MainForm.cs b/UI.WinForms.Msil/Forms/MainForm.cs index 2af172a..dae9e60 100644 --- a/UI.WinForms.Msil/Forms/MainForm.cs +++ b/UI.WinForms.Msil/Forms/MainForm.cs @@ -417,12 +417,12 @@ private void DumpDiscFinished(object sender, RunWorkerCompletedEventArgs e) rescanDiscsButton.Enabled = true; rescanDiscsButton.Visible = true; - if (dumper.BrokenFiles.Any()) + if (dumper.ValidationStatus == false) { step4StatusLabel.Text = "❌"; step4Label.Text = "Dump is corrupted"; } - else if (dumper.ValidatingDiscKeys.Any()) + else if (dumper.ValidationStatus == true) { step4StatusLabel.Text = "✔"; step4Label.Text = "Dump is valid";